| @@ -1,1321 +1,1324 | |||||
| 1 | ''' |
|
1 | ''' | |
| 2 | Created on September , 2012 |
|
2 | Created on September , 2012 | |
| 3 | @author: |
|
3 | @author: | |
| 4 | ''' |
|
4 | ''' | |
| 5 |
|
5 | |||
| 6 | import sys |
|
6 | import sys | |
| 7 | import ast |
|
7 | import ast | |
| 8 | import datetime |
|
8 | import datetime | |
| 9 | import traceback |
|
9 | import traceback | |
| 10 | import math |
|
10 | import math | |
| 11 | import time |
|
11 | import time | |
| 12 | from multiprocessing import Process, Queue, cpu_count |
|
12 | from multiprocessing import Process, Queue, cpu_count | |
| 13 |
|
13 | |||
| 14 | import schainpy |
|
14 | import schainpy | |
| 15 | import schainpy.admin |
|
15 | import schainpy.admin | |
| 16 |
|
16 | |||
| 17 | from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring |
|
17 | from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring | |
| 18 | from xml.dom import minidom |
|
18 | from xml.dom import minidom | |
| 19 |
|
19 | |||
| 20 | from schainpy.model import * |
|
20 | from schainpy.model import * | |
| 21 | from time import sleep |
|
21 | from time import sleep | |
| 22 |
|
22 | |||
| 23 | def prettify(elem): |
|
23 | def prettify(elem): | |
| 24 | """Return a pretty-printed XML string for the Element. |
|
24 | """Return a pretty-printed XML string for the Element. | |
| 25 | """ |
|
25 | """ | |
| 26 | rough_string = tostring(elem, 'utf-8') |
|
26 | rough_string = tostring(elem, 'utf-8') | |
| 27 | reparsed = minidom.parseString(rough_string) |
|
27 | reparsed = minidom.parseString(rough_string) | |
| 28 | return reparsed.toprettyxml(indent=" ") |
|
28 | return reparsed.toprettyxml(indent=" ") | |
| 29 |
|
29 | |||
| 30 | def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False): |
|
30 | def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False): | |
| 31 | skip = 0 |
|
31 | skip = 0 | |
| 32 | cursor = 0 |
|
32 | cursor = 0 | |
| 33 | nFiles = None |
|
33 | nFiles = None | |
| 34 | processes = [] |
|
34 | processes = [] | |
| 35 | dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d') |
|
35 | dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d') | |
| 36 | dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d') |
|
36 | dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d') | |
| 37 | days = (dt2 - dt1).days |
|
37 | days = (dt2 - dt1).days | |
| 38 |
|
38 | |||
| 39 | for day in range(days+1): |
|
39 | for day in range(days+1): | |
| 40 | skip = 0 |
|
40 | skip = 0 | |
| 41 | cursor = 0 |
|
41 | cursor = 0 | |
| 42 | q = Queue() |
|
42 | q = Queue() | |
| 43 | processes = [] |
|
43 | processes = [] | |
| 44 | dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d') |
|
44 | dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d') | |
| 45 | firstProcess = Process(target=child, args=(cursor, skip, q, dt)) |
|
45 | firstProcess = Process(target=child, args=(cursor, skip, q, dt)) | |
| 46 | firstProcess.start() |
|
46 | firstProcess.start() | |
| 47 | if by_day: |
|
47 | if by_day: | |
| 48 | continue |
|
48 | continue | |
| 49 | nFiles = q.get() |
|
49 | nFiles = q.get() | |
| 50 | firstProcess.terminate() |
|
50 | firstProcess.terminate() | |
| 51 | skip = int(math.ceil(nFiles/nProcess)) |
|
51 | skip = int(math.ceil(nFiles/nProcess)) | |
| 52 | while True: |
|
52 | while True: | |
| 53 | processes.append(Process(target=child, args=(cursor, skip, q, dt))) |
|
53 | processes.append(Process(target=child, args=(cursor, skip, q, dt))) | |
| 54 | processes[cursor].start() |
|
54 | processes[cursor].start() | |
| 55 | if nFiles < cursor*skip: |
|
55 | if nFiles < cursor*skip: | |
| 56 | break |
|
56 | break | |
| 57 | cursor += 1 |
|
57 | cursor += 1 | |
| 58 |
|
58 | |||
| 59 | def beforeExit(exctype, value, trace): |
|
59 | def beforeExit(exctype, value, trace): | |
| 60 | for process in processes: |
|
60 | for process in processes: | |
| 61 | process.terminate() |
|
61 | process.terminate() | |
| 62 | process.join() |
|
62 | process.join() | |
| 63 | print traceback.print_tb(trace) |
|
63 | print traceback.print_tb(trace) | |
| 64 |
|
64 | |||
| 65 | sys.excepthook = beforeExit |
|
65 | sys.excepthook = beforeExit | |
| 66 |
|
66 | |||
| 67 | for process in processes: |
|
67 | for process in processes: | |
| 68 | process.join() |
|
68 | process.join() | |
| 69 | process.terminate() |
|
69 | process.terminate() | |
| 70 | time.sleep(3) |
|
70 | time.sleep(3) | |
| 71 |
|
71 | |||
| 72 | class ParameterConf(): |
|
72 | class ParameterConf(): | |
| 73 |
|
73 | |||
| 74 | id = None |
|
74 | id = None | |
| 75 | name = None |
|
75 | name = None | |
| 76 | value = None |
|
76 | value = None | |
| 77 | format = None |
|
77 | format = None | |
| 78 |
|
78 | |||
| 79 | __formated_value = None |
|
79 | __formated_value = None | |
| 80 |
|
80 | |||
| 81 | ELEMENTNAME = 'Parameter' |
|
81 | ELEMENTNAME = 'Parameter' | |
| 82 |
|
82 | |||
| 83 | def __init__(self): |
|
83 | def __init__(self): | |
| 84 |
|
84 | |||
| 85 | self.format = 'str' |
|
85 | self.format = 'str' | |
| 86 |
|
86 | |||
| 87 | def getElementName(self): |
|
87 | def getElementName(self): | |
| 88 |
|
88 | |||
| 89 | return self.ELEMENTNAME |
|
89 | return self.ELEMENTNAME | |
| 90 |
|
90 | |||
| 91 | def getValue(self): |
|
91 | def getValue(self): | |
| 92 |
|
92 | |||
| 93 | value = self.value |
|
93 | value = self.value | |
| 94 | format = self.format |
|
94 | format = self.format | |
| 95 |
|
95 | |||
| 96 | if self.__formated_value != None: |
|
96 | if self.__formated_value != None: | |
| 97 |
|
97 | |||
| 98 | return self.__formated_value |
|
98 | return self.__formated_value | |
| 99 |
|
99 | |||
| 100 | if format == 'obj': |
|
100 | if format == 'obj': | |
| 101 | return value |
|
101 | return value | |
| 102 |
|
102 | |||
| 103 | if format == 'str': |
|
103 | if format == 'str': | |
| 104 | self.__formated_value = str(value) |
|
104 | self.__formated_value = str(value) | |
| 105 | return self.__formated_value |
|
105 | return self.__formated_value | |
| 106 |
|
106 | |||
| 107 | if value == '': |
|
107 | if value == '': | |
| 108 | raise ValueError, "%s: This parameter value is empty" %self.name |
|
108 | raise ValueError, "%s: This parameter value is empty" %self.name | |
| 109 |
|
109 | |||
| 110 | if format == 'list': |
|
110 | if format == 'list': | |
| 111 | strList = value.split(',') |
|
111 | strList = value.split(',') | |
| 112 |
|
112 | |||
| 113 | self.__formated_value = strList |
|
113 | self.__formated_value = strList | |
| 114 |
|
114 | |||
| 115 | return self.__formated_value |
|
115 | return self.__formated_value | |
| 116 |
|
116 | |||
| 117 | if format == 'intlist': |
|
117 | if format == 'intlist': | |
| 118 | """ |
|
118 | """ | |
| 119 | Example: |
|
119 | Example: | |
| 120 | value = (0,1,2) |
|
120 | value = (0,1,2) | |
| 121 | """ |
|
121 | """ | |
| 122 |
|
122 | |||
| 123 | new_value = ast.literal_eval(value) |
|
123 | new_value = ast.literal_eval(value) | |
| 124 |
|
124 | |||
| 125 | if type(new_value) not in (tuple, list): |
|
125 | if type(new_value) not in (tuple, list): | |
| 126 | new_value = [int(new_value)] |
|
126 | new_value = [int(new_value)] | |
| 127 |
|
127 | |||
| 128 | self.__formated_value = new_value |
|
128 | self.__formated_value = new_value | |
| 129 |
|
129 | |||
| 130 | return self.__formated_value |
|
130 | return self.__formated_value | |
| 131 |
|
131 | |||
| 132 | if format == 'floatlist': |
|
132 | if format == 'floatlist': | |
| 133 | """ |
|
133 | """ | |
| 134 | Example: |
|
134 | Example: | |
| 135 | value = (0.5, 1.4, 2.7) |
|
135 | value = (0.5, 1.4, 2.7) | |
| 136 | """ |
|
136 | """ | |
| 137 |
|
137 | |||
| 138 | new_value = ast.literal_eval(value) |
|
138 | new_value = ast.literal_eval(value) | |
| 139 |
|
139 | |||
| 140 | if type(new_value) not in (tuple, list): |
|
140 | if type(new_value) not in (tuple, list): | |
| 141 | new_value = [float(new_value)] |
|
141 | new_value = [float(new_value)] | |
| 142 |
|
142 | |||
| 143 | self.__formated_value = new_value |
|
143 | self.__formated_value = new_value | |
| 144 |
|
144 | |||
| 145 | return self.__formated_value |
|
145 | return self.__formated_value | |
| 146 |
|
146 | |||
| 147 | if format == 'date': |
|
147 | if format == 'date': | |
| 148 | strList = value.split('/') |
|
148 | strList = value.split('/') | |
| 149 | intList = [int(x) for x in strList] |
|
149 | intList = [int(x) for x in strList] | |
| 150 | date = datetime.date(intList[0], intList[1], intList[2]) |
|
150 | date = datetime.date(intList[0], intList[1], intList[2]) | |
| 151 |
|
151 | |||
| 152 | self.__formated_value = date |
|
152 | self.__formated_value = date | |
| 153 |
|
153 | |||
| 154 | return self.__formated_value |
|
154 | return self.__formated_value | |
| 155 |
|
155 | |||
| 156 | if format == 'time': |
|
156 | if format == 'time': | |
| 157 | strList = value.split(':') |
|
157 | strList = value.split(':') | |
| 158 | intList = [int(x) for x in strList] |
|
158 | intList = [int(x) for x in strList] | |
| 159 | time = datetime.time(intList[0], intList[1], intList[2]) |
|
159 | time = datetime.time(intList[0], intList[1], intList[2]) | |
| 160 |
|
160 | |||
| 161 | self.__formated_value = time |
|
161 | self.__formated_value = time | |
| 162 |
|
162 | |||
| 163 | return self.__formated_value |
|
163 | return self.__formated_value | |
| 164 |
|
164 | |||
| 165 | if format == 'pairslist': |
|
165 | if format == 'pairslist': | |
| 166 | """ |
|
166 | """ | |
| 167 | Example: |
|
167 | Example: | |
| 168 | value = (0,1),(1,2) |
|
168 | value = (0,1),(1,2) | |
| 169 | """ |
|
169 | """ | |
| 170 |
|
170 | |||
| 171 | new_value = ast.literal_eval(value) |
|
171 | new_value = ast.literal_eval(value) | |
| 172 |
|
172 | |||
| 173 | if type(new_value) not in (tuple, list): |
|
173 | if type(new_value) not in (tuple, list): | |
| 174 | raise ValueError, "%s has to be a tuple or list of pairs" %value |
|
174 | raise ValueError, "%s has to be a tuple or list of pairs" %value | |
| 175 |
|
175 | |||
| 176 | if type(new_value[0]) not in (tuple, list): |
|
176 | if type(new_value[0]) not in (tuple, list): | |
| 177 | if len(new_value) != 2: |
|
177 | if len(new_value) != 2: | |
| 178 | raise ValueError, "%s has to be a tuple or list of pairs" %value |
|
178 | raise ValueError, "%s has to be a tuple or list of pairs" %value | |
| 179 | new_value = [new_value] |
|
179 | new_value = [new_value] | |
| 180 |
|
180 | |||
| 181 | for thisPair in new_value: |
|
181 | for thisPair in new_value: | |
| 182 | if len(thisPair) != 2: |
|
182 | if len(thisPair) != 2: | |
| 183 | raise ValueError, "%s has to be a tuple or list of pairs" %value |
|
183 | raise ValueError, "%s has to be a tuple or list of pairs" %value | |
| 184 |
|
184 | |||
| 185 | self.__formated_value = new_value |
|
185 | self.__formated_value = new_value | |
| 186 |
|
186 | |||
| 187 | return self.__formated_value |
|
187 | return self.__formated_value | |
| 188 |
|
188 | |||
| 189 | if format == 'multilist': |
|
189 | if format == 'multilist': | |
| 190 | """ |
|
190 | """ | |
| 191 | Example: |
|
191 | Example: | |
| 192 | value = (0,1,2),(3,4,5) |
|
192 | value = (0,1,2),(3,4,5) | |
| 193 | """ |
|
193 | """ | |
| 194 | multiList = ast.literal_eval(value) |
|
194 | multiList = ast.literal_eval(value) | |
| 195 |
|
195 | |||
| 196 | if type(multiList[0]) == int: |
|
196 | if type(multiList[0]) == int: | |
| 197 | multiList = ast.literal_eval("(" + value + ")") |
|
197 | multiList = ast.literal_eval("(" + value + ")") | |
| 198 |
|
198 | |||
| 199 | self.__formated_value = multiList |
|
199 | self.__formated_value = multiList | |
| 200 |
|
200 | |||
| 201 | return self.__formated_value |
|
201 | return self.__formated_value | |
| 202 |
|
202 | |||
| 203 | if format == 'bool': |
|
203 | if format == 'bool': | |
| 204 | value = int(value) |
|
204 | value = int(value) | |
| 205 |
|
205 | |||
| 206 | if format == 'int': |
|
206 | if format == 'int': | |
| 207 | value = float(value) |
|
207 | value = float(value) | |
| 208 |
|
208 | |||
| 209 | format_func = eval(format) |
|
209 | format_func = eval(format) | |
| 210 |
|
210 | |||
| 211 | self.__formated_value = format_func(value) |
|
211 | self.__formated_value = format_func(value) | |
| 212 |
|
212 | |||
| 213 | return self.__formated_value |
|
213 | return self.__formated_value | |
| 214 |
|
214 | |||
| 215 | def updateId(self, new_id): |
|
215 | def updateId(self, new_id): | |
| 216 |
|
216 | |||
| 217 | self.id = str(new_id) |
|
217 | self.id = str(new_id) | |
| 218 |
|
218 | |||
| 219 | def setup(self, id, name, value, format='str'): |
|
219 | def setup(self, id, name, value, format='str'): | |
| 220 |
|
||||
| 221 | self.id = str(id) |
|
220 | self.id = str(id) | |
| 222 | self.name = name |
|
221 | self.name = name | |
| 223 | if format == 'obj': |
|
222 | if format == 'obj': | |
| 224 | self.value = value |
|
223 | self.value = value | |
| 225 | else: |
|
224 | else: | |
| 226 | self.value = str(value) |
|
225 | self.value = str(value) | |
| 227 | self.format = str.lower(format) |
|
226 | self.format = str.lower(format) | |
| 228 |
|
227 | |||
| 229 | self.getValue() |
|
228 | self.getValue() | |
| 230 |
|
229 | |||
| 231 | return 1 |
|
230 | return 1 | |
| 232 |
|
231 | |||
| 233 | def update(self, name, value, format='str'): |
|
232 | def update(self, name, value, format='str'): | |
| 234 |
|
233 | |||
| 235 | self.name = name |
|
234 | self.name = name | |
| 236 | self.value = str(value) |
|
235 | self.value = str(value) | |
| 237 | self.format = format |
|
236 | self.format = format | |
| 238 |
|
237 | |||
| 239 | def makeXml(self, opElement): |
|
238 | def makeXml(self, opElement): | |
| 240 | if self.name not in ('queue',): |
|
239 | if self.name not in ('queue',): | |
| 241 | parmElement = SubElement(opElement, self.ELEMENTNAME) |
|
240 | parmElement = SubElement(opElement, self.ELEMENTNAME) | |
| 242 | parmElement.set('id', str(self.id)) |
|
241 | parmElement.set('id', str(self.id)) | |
| 243 | parmElement.set('name', self.name) |
|
242 | parmElement.set('name', self.name) | |
| 244 | parmElement.set('value', self.value) |
|
243 | parmElement.set('value', self.value) | |
| 245 | parmElement.set('format', self.format) |
|
244 | parmElement.set('format', self.format) | |
| 246 |
|
245 | |||
| 247 | def readXml(self, parmElement): |
|
246 | def readXml(self, parmElement): | |
| 248 |
|
247 | |||
| 249 | self.id = parmElement.get('id') |
|
248 | self.id = parmElement.get('id') | |
| 250 | self.name = parmElement.get('name') |
|
249 | self.name = parmElement.get('name') | |
| 251 | self.value = parmElement.get('value') |
|
250 | self.value = parmElement.get('value') | |
| 252 | self.format = str.lower(parmElement.get('format')) |
|
251 | self.format = str.lower(parmElement.get('format')) | |
| 253 |
|
252 | |||
| 254 | #Compatible with old signal chain version |
|
253 | #Compatible with old signal chain version | |
| 255 | if self.format == 'int' and self.name == 'idfigure': |
|
254 | if self.format == 'int' and self.name == 'idfigure': | |
| 256 | self.name = 'id' |
|
255 | self.name = 'id' | |
| 257 |
|
256 | |||
| 258 | def printattr(self): |
|
257 | def printattr(self): | |
| 259 |
|
258 | |||
| 260 | print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format) |
|
259 | print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format) | |
| 261 |
|
260 | |||
| 262 | class OperationConf(): |
|
261 | class OperationConf(): | |
| 263 |
|
262 | |||
| 264 | id = None |
|
263 | id = None | |
| 265 | name = None |
|
264 | name = None | |
| 266 | priority = None |
|
265 | priority = None | |
| 267 | type = None |
|
266 | type = None | |
| 268 |
|
267 | |||
| 269 | parmConfObjList = [] |
|
268 | parmConfObjList = [] | |
| 270 |
|
269 | |||
| 271 | ELEMENTNAME = 'Operation' |
|
270 | ELEMENTNAME = 'Operation' | |
| 272 |
|
271 | |||
| 273 | def __init__(self): |
|
272 | def __init__(self): | |
| 274 |
|
273 | |||
| 275 | self.id = '0' |
|
274 | self.id = '0' | |
| 276 | self.name = None |
|
275 | self.name = None | |
| 277 | self.priority = None |
|
276 | self.priority = None | |
| 278 | self.type = 'self' |
|
277 | self.type = 'self' | |
| 279 |
|
278 | |||
| 280 |
|
279 | |||
| 281 | def __getNewId(self): |
|
280 | def __getNewId(self): | |
| 282 |
|
281 | |||
| 283 | return int(self.id)*10 + len(self.parmConfObjList) + 1 |
|
282 | return int(self.id)*10 + len(self.parmConfObjList) + 1 | |
| 284 |
|
283 | |||
| 285 | def updateId(self, new_id): |
|
284 | def updateId(self, new_id): | |
| 286 |
|
285 | |||
| 287 | self.id = str(new_id) |
|
286 | self.id = str(new_id) | |
| 288 |
|
287 | |||
| 289 | n = 1 |
|
288 | n = 1 | |
| 290 | for parmObj in self.parmConfObjList: |
|
289 | for parmObj in self.parmConfObjList: | |
| 291 |
|
290 | |||
| 292 | idParm = str(int(new_id)*10 + n) |
|
291 | idParm = str(int(new_id)*10 + n) | |
| 293 | parmObj.updateId(idParm) |
|
292 | parmObj.updateId(idParm) | |
| 294 |
|
293 | |||
| 295 | n += 1 |
|
294 | n += 1 | |
| 296 |
|
295 | |||
| 297 | def getElementName(self): |
|
296 | def getElementName(self): | |
| 298 |
|
297 | |||
| 299 | return self.ELEMENTNAME |
|
298 | return self.ELEMENTNAME | |
| 300 |
|
299 | |||
| 301 | def getParameterObjList(self): |
|
300 | def getParameterObjList(self): | |
| 302 |
|
301 | |||
| 303 | return self.parmConfObjList |
|
302 | return self.parmConfObjList | |
| 304 |
|
303 | |||
| 305 | def getParameterObj(self, parameterName): |
|
304 | def getParameterObj(self, parameterName): | |
| 306 |
|
305 | |||
| 307 | for parmConfObj in self.parmConfObjList: |
|
306 | for parmConfObj in self.parmConfObjList: | |
| 308 |
|
307 | |||
| 309 | if parmConfObj.name != parameterName: |
|
308 | if parmConfObj.name != parameterName: | |
| 310 | continue |
|
309 | continue | |
| 311 |
|
310 | |||
| 312 | return parmConfObj |
|
311 | return parmConfObj | |
| 313 |
|
312 | |||
| 314 | return None |
|
313 | return None | |
| 315 |
|
314 | |||
| 316 | def getParameterObjfromValue(self, parameterValue): |
|
315 | def getParameterObjfromValue(self, parameterValue): | |
| 317 |
|
316 | |||
| 318 | for parmConfObj in self.parmConfObjList: |
|
317 | for parmConfObj in self.parmConfObjList: | |
| 319 |
|
318 | |||
| 320 | if parmConfObj.getValue() != parameterValue: |
|
319 | if parmConfObj.getValue() != parameterValue: | |
| 321 | continue |
|
320 | continue | |
| 322 |
|
321 | |||
| 323 | return parmConfObj.getValue() |
|
322 | return parmConfObj.getValue() | |
| 324 |
|
323 | |||
| 325 | return None |
|
324 | return None | |
| 326 |
|
325 | |||
| 327 | def getParameterValue(self, parameterName): |
|
326 | def getParameterValue(self, parameterName): | |
| 328 |
|
327 | |||
| 329 | parameterObj = self.getParameterObj(parameterName) |
|
328 | parameterObj = self.getParameterObj(parameterName) | |
| 330 |
|
329 | |||
| 331 | # if not parameterObj: |
|
330 | # if not parameterObj: | |
| 332 | # return None |
|
331 | # return None | |
| 333 |
|
332 | |||
| 334 | value = parameterObj.getValue() |
|
333 | value = parameterObj.getValue() | |
| 335 |
|
334 | |||
| 336 | return value |
|
335 | return value | |
| 337 |
|
336 | |||
| 338 |
|
337 | |||
| 339 | def getKwargs(self): |
|
338 | def getKwargs(self): | |
| 340 |
|
339 | |||
| 341 | kwargs = {} |
|
340 | kwargs = {} | |
| 342 |
|
341 | |||
| 343 | for parmConfObj in self.parmConfObjList: |
|
342 | for parmConfObj in self.parmConfObjList: | |
| 344 | if self.name == 'run' and parmConfObj.name == 'datatype': |
|
343 | if self.name == 'run' and parmConfObj.name == 'datatype': | |
| 345 | continue |
|
344 | continue | |
| 346 |
|
345 | |||
| 347 | kwargs[parmConfObj.name] = parmConfObj.getValue() |
|
346 | kwargs[parmConfObj.name] = parmConfObj.getValue() | |
| 348 |
|
347 | |||
| 349 | return kwargs |
|
348 | return kwargs | |
| 350 |
|
349 | |||
| 351 | def setup(self, id, name, priority, type): |
|
350 | def setup(self, id, name, priority, type): | |
| 352 |
|
351 | |||
| 353 | self.id = str(id) |
|
352 | self.id = str(id) | |
| 354 | self.name = name |
|
353 | self.name = name | |
| 355 | self.type = type |
|
354 | self.type = type | |
| 356 | self.priority = priority |
|
355 | self.priority = priority | |
| 357 |
|
356 | |||
| 358 | self.parmConfObjList = [] |
|
357 | self.parmConfObjList = [] | |
| 359 |
|
358 | |||
| 360 | def removeParameters(self): |
|
359 | def removeParameters(self): | |
| 361 |
|
360 | |||
| 362 | for obj in self.parmConfObjList: |
|
361 | for obj in self.parmConfObjList: | |
| 363 | del obj |
|
362 | del obj | |
| 364 |
|
363 | |||
| 365 | self.parmConfObjList = [] |
|
364 | self.parmConfObjList = [] | |
| 366 |
|
365 | |||
| 367 | def addParameter(self, name, value, format='str'): |
|
366 | def addParameter(self, name, value, format='str'): | |
| 368 |
|
367 | |||
| 369 | id = self.__getNewId() |
|
368 | id = self.__getNewId() | |
| 370 |
|
369 | |||
| 371 | parmConfObj = ParameterConf() |
|
370 | parmConfObj = ParameterConf() | |
| 372 | if not parmConfObj.setup(id, name, value, format): |
|
371 | if not parmConfObj.setup(id, name, value, format): | |
| 373 | return None |
|
372 | return None | |
| 374 |
|
373 | |||
| 375 | self.parmConfObjList.append(parmConfObj) |
|
374 | self.parmConfObjList.append(parmConfObj) | |
| 376 |
|
375 | |||
| 377 | return parmConfObj |
|
376 | return parmConfObj | |
| 378 |
|
377 | |||
| 379 | def changeParameter(self, name, value, format='str'): |
|
378 | def changeParameter(self, name, value, format='str'): | |
| 380 |
|
379 | |||
| 381 | parmConfObj = self.getParameterObj(name) |
|
380 | parmConfObj = self.getParameterObj(name) | |
| 382 | parmConfObj.update(name, value, format) |
|
381 | parmConfObj.update(name, value, format) | |
| 383 |
|
382 | |||
| 384 | return parmConfObj |
|
383 | return parmConfObj | |
| 385 |
|
384 | |||
| 386 | def makeXml(self, procUnitElement): |
|
385 | def makeXml(self, procUnitElement): | |
| 387 |
|
386 | |||
| 388 | opElement = SubElement(procUnitElement, self.ELEMENTNAME) |
|
387 | opElement = SubElement(procUnitElement, self.ELEMENTNAME) | |
| 389 | opElement.set('id', str(self.id)) |
|
388 | opElement.set('id', str(self.id)) | |
| 390 | opElement.set('name', self.name) |
|
389 | opElement.set('name', self.name) | |
| 391 | opElement.set('type', self.type) |
|
390 | opElement.set('type', self.type) | |
| 392 | opElement.set('priority', str(self.priority)) |
|
391 | opElement.set('priority', str(self.priority)) | |
| 393 |
|
392 | |||
| 394 | for parmConfObj in self.parmConfObjList: |
|
393 | for parmConfObj in self.parmConfObjList: | |
| 395 | parmConfObj.makeXml(opElement) |
|
394 | parmConfObj.makeXml(opElement) | |
| 396 |
|
395 | |||
| 397 | def readXml(self, opElement): |
|
396 | def readXml(self, opElement): | |
| 398 |
|
397 | |||
| 399 | self.id = opElement.get('id') |
|
398 | self.id = opElement.get('id') | |
| 400 | self.name = opElement.get('name') |
|
399 | self.name = opElement.get('name') | |
| 401 | self.type = opElement.get('type') |
|
400 | self.type = opElement.get('type') | |
| 402 | self.priority = opElement.get('priority') |
|
401 | self.priority = opElement.get('priority') | |
| 403 |
|
402 | |||
| 404 | #Compatible with old signal chain version |
|
403 | #Compatible with old signal chain version | |
| 405 | #Use of 'run' method instead 'init' |
|
404 | #Use of 'run' method instead 'init' | |
| 406 | if self.type == 'self' and self.name == 'init': |
|
405 | if self.type == 'self' and self.name == 'init': | |
| 407 | self.name = 'run' |
|
406 | self.name = 'run' | |
| 408 |
|
407 | |||
| 409 | self.parmConfObjList = [] |
|
408 | self.parmConfObjList = [] | |
| 410 |
|
409 | |||
| 411 | parmElementList = opElement.iter(ParameterConf().getElementName()) |
|
410 | parmElementList = opElement.iter(ParameterConf().getElementName()) | |
| 412 |
|
411 | |||
| 413 | for parmElement in parmElementList: |
|
412 | for parmElement in parmElementList: | |
| 414 | parmConfObj = ParameterConf() |
|
413 | parmConfObj = ParameterConf() | |
| 415 | parmConfObj.readXml(parmElement) |
|
414 | parmConfObj.readXml(parmElement) | |
| 416 |
|
415 | |||
| 417 | #Compatible with old signal chain version |
|
416 | #Compatible with old signal chain version | |
| 418 | #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER |
|
417 | #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER | |
| 419 | if self.type != 'self' and self.name == 'Plot': |
|
418 | if self.type != 'self' and self.name == 'Plot': | |
| 420 | if parmConfObj.format == 'str' and parmConfObj.name == 'type': |
|
419 | if parmConfObj.format == 'str' and parmConfObj.name == 'type': | |
| 421 | self.name = parmConfObj.value |
|
420 | self.name = parmConfObj.value | |
| 422 | continue |
|
421 | continue | |
| 423 |
|
422 | |||
| 424 | self.parmConfObjList.append(parmConfObj) |
|
423 | self.parmConfObjList.append(parmConfObj) | |
| 425 |
|
424 | |||
| 426 | def printattr(self): |
|
425 | def printattr(self): | |
| 427 |
|
426 | |||
| 428 | print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME, |
|
427 | print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME, | |
| 429 | self.id, |
|
428 | self.id, | |
| 430 | self.name, |
|
429 | self.name, | |
| 431 | self.type, |
|
430 | self.type, | |
| 432 | self.priority) |
|
431 | self.priority) | |
| 433 |
|
432 | |||
| 434 | for parmConfObj in self.parmConfObjList: |
|
433 | for parmConfObj in self.parmConfObjList: | |
| 435 | parmConfObj.printattr() |
|
434 | parmConfObj.printattr() | |
| 436 |
|
435 | |||
| 437 | def createObject(self, plotter_queue=None): |
|
436 | def createObject(self, plotter_queue=None): | |
| 438 |
|
437 | |||
| 439 |
|
438 | |||
| 440 | if self.type == 'self': |
|
439 | if self.type == 'self': | |
| 441 | raise ValueError, "This operation type cannot be created" |
|
440 | raise ValueError, "This operation type cannot be created" | |
| 442 |
|
441 | |||
| 443 | if self.type == 'plotter': |
|
442 | if self.type == 'plotter': | |
| 444 | #Plotter(plotter_name) |
|
443 | #Plotter(plotter_name) | |
| 445 | if not plotter_queue: |
|
444 | if not plotter_queue: | |
| 446 | raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)" |
|
445 | raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)" | |
| 447 |
|
446 | |||
| 448 | opObj = Plotter(self.name, plotter_queue) |
|
447 | opObj = Plotter(self.name, plotter_queue) | |
| 449 |
|
448 | |||
| 450 | if self.type == 'external' or self.type == 'other': |
|
449 | if self.type == 'external' or self.type == 'other': | |
| 451 |
|
450 | |||
| 452 | className = eval(self.name) |
|
451 | className = eval(self.name) | |
| 453 | kwargs = self.getKwargs() |
|
452 | kwargs = self.getKwargs() | |
| 454 |
|
453 | |||
| 455 | opObj = className(**kwargs) |
|
454 | opObj = className(**kwargs) | |
| 456 |
|
455 | |||
| 457 | return opObj |
|
456 | return opObj | |
| 458 |
|
457 | |||
| 459 |
|
458 | |||
| 460 | class ProcUnitConf(): |
|
459 | class ProcUnitConf(): | |
| 461 |
|
460 | |||
| 462 | id = None |
|
461 | id = None | |
| 463 | name = None |
|
462 | name = None | |
| 464 | datatype = None |
|
463 | datatype = None | |
| 465 | inputId = None |
|
464 | inputId = None | |
| 466 | parentId = None |
|
465 | parentId = None | |
| 467 |
|
466 | |||
| 468 | opConfObjList = [] |
|
467 | opConfObjList = [] | |
| 469 |
|
468 | |||
| 470 | procUnitObj = None |
|
469 | procUnitObj = None | |
| 471 | opObjList = [] |
|
470 | opObjList = [] | |
| 472 |
|
471 | |||
| 473 | ELEMENTNAME = 'ProcUnit' |
|
472 | ELEMENTNAME = 'ProcUnit' | |
| 474 |
|
473 | |||
| 475 | def __init__(self): |
|
474 | def __init__(self): | |
| 476 |
|
475 | |||
| 477 | self.id = None |
|
476 | self.id = None | |
| 478 | self.datatype = None |
|
477 | self.datatype = None | |
| 479 | self.name = None |
|
478 | self.name = None | |
| 480 | self.inputId = None |
|
479 | self.inputId = None | |
| 481 |
|
480 | |||
| 482 | self.opConfObjList = [] |
|
481 | self.opConfObjList = [] | |
| 483 |
|
482 | |||
| 484 | self.procUnitObj = None |
|
483 | self.procUnitObj = None | |
| 485 | self.opObjDict = {} |
|
484 | self.opObjDict = {} | |
| 486 |
|
485 | |||
| 487 | def __getPriority(self): |
|
486 | def __getPriority(self): | |
| 488 |
|
487 | |||
| 489 | return len(self.opConfObjList)+1 |
|
488 | return len(self.opConfObjList)+1 | |
| 490 |
|
489 | |||
| 491 | def __getNewId(self): |
|
490 | def __getNewId(self): | |
| 492 |
|
491 | |||
| 493 | return int(self.id)*10 + len(self.opConfObjList) + 1 |
|
492 | return int(self.id)*10 + len(self.opConfObjList) + 1 | |
| 494 |
|
493 | |||
| 495 | def getElementName(self): |
|
494 | def getElementName(self): | |
| 496 |
|
495 | |||
| 497 | return self.ELEMENTNAME |
|
496 | return self.ELEMENTNAME | |
| 498 |
|
497 | |||
| 499 | def getId(self): |
|
498 | def getId(self): | |
| 500 |
|
499 | |||
| 501 | return self.id |
|
500 | return self.id | |
| 502 |
|
501 | |||
| 503 | def updateId(self, new_id, parentId=parentId): |
|
502 | def updateId(self, new_id, parentId=parentId): | |
| 504 |
|
503 | |||
| 505 |
|
504 | |||
| 506 | new_id = int(parentId)*10 + (int(self.id) % 10) |
|
505 | new_id = int(parentId)*10 + (int(self.id) % 10) | |
| 507 | new_inputId = int(parentId)*10 + (int(self.inputId) % 10) |
|
506 | new_inputId = int(parentId)*10 + (int(self.inputId) % 10) | |
| 508 |
|
507 | |||
| 509 | #If this proc unit has not inputs |
|
508 | #If this proc unit has not inputs | |
| 510 | if self.inputId == '0': |
|
509 | if self.inputId == '0': | |
| 511 | new_inputId = 0 |
|
510 | new_inputId = 0 | |
| 512 |
|
511 | |||
| 513 | n = 1 |
|
512 | n = 1 | |
| 514 | for opConfObj in self.opConfObjList: |
|
513 | for opConfObj in self.opConfObjList: | |
| 515 |
|
514 | |||
| 516 | idOp = str(int(new_id)*10 + n) |
|
515 | idOp = str(int(new_id)*10 + n) | |
| 517 | opConfObj.updateId(idOp) |
|
516 | opConfObj.updateId(idOp) | |
| 518 |
|
517 | |||
| 519 | n += 1 |
|
518 | n += 1 | |
| 520 |
|
519 | |||
| 521 | self.parentId = str(parentId) |
|
520 | self.parentId = str(parentId) | |
| 522 | self.id = str(new_id) |
|
521 | self.id = str(new_id) | |
| 523 | self.inputId = str(new_inputId) |
|
522 | self.inputId = str(new_inputId) | |
| 524 |
|
523 | |||
| 525 |
|
524 | |||
| 526 | def getInputId(self): |
|
525 | def getInputId(self): | |
| 527 |
|
526 | |||
| 528 | return self.inputId |
|
527 | return self.inputId | |
| 529 |
|
528 | |||
| 530 | def getOperationObjList(self): |
|
529 | def getOperationObjList(self): | |
| 531 |
|
530 | |||
| 532 | return self.opConfObjList |
|
531 | return self.opConfObjList | |
| 533 |
|
532 | |||
| 534 | def getOperationObj(self, name=None): |
|
533 | def getOperationObj(self, name=None): | |
| 535 |
|
534 | |||
| 536 | for opConfObj in self.opConfObjList: |
|
535 | for opConfObj in self.opConfObjList: | |
| 537 |
|
536 | |||
| 538 | if opConfObj.name != name: |
|
537 | if opConfObj.name != name: | |
| 539 | continue |
|
538 | continue | |
| 540 |
|
539 | |||
| 541 | return opConfObj |
|
540 | return opConfObj | |
| 542 |
|
541 | |||
| 543 | return None |
|
542 | return None | |
| 544 |
|
543 | |||
| 545 | def getOpObjfromParamValue(self, value=None): |
|
544 | def getOpObjfromParamValue(self, value=None): | |
| 546 |
|
545 | |||
| 547 | for opConfObj in self.opConfObjList: |
|
546 | for opConfObj in self.opConfObjList: | |
| 548 | if opConfObj.getParameterObjfromValue(parameterValue=value) != value: |
|
547 | if opConfObj.getParameterObjfromValue(parameterValue=value) != value: | |
| 549 | continue |
|
548 | continue | |
| 550 | return opConfObj |
|
549 | return opConfObj | |
| 551 | return None |
|
550 | return None | |
| 552 |
|
551 | |||
| 553 | def getProcUnitObj(self): |
|
552 | def getProcUnitObj(self): | |
| 554 |
|
553 | |||
| 555 | return self.procUnitObj |
|
554 | return self.procUnitObj | |
| 556 |
|
555 | |||
| 557 | def setup(self, id, name, datatype, inputId, parentId=None): |
|
556 | def setup(self, id, name, datatype, inputId, parentId=None): | |
| 558 |
|
557 | |||
| 559 | #Compatible with old signal chain version |
|
558 | #Compatible with old signal chain version | |
| 560 | if datatype==None and name==None: |
|
559 | if datatype==None and name==None: | |
| 561 | raise ValueError, "datatype or name should be defined" |
|
560 | raise ValueError, "datatype or name should be defined" | |
| 562 |
|
561 | |||
| 563 | if name==None: |
|
562 | if name==None: | |
| 564 | if 'Proc' in datatype: |
|
563 | if 'Proc' in datatype: | |
| 565 | name = datatype |
|
564 | name = datatype | |
| 566 | else: |
|
565 | else: | |
| 567 | name = '%sProc' %(datatype) |
|
566 | name = '%sProc' %(datatype) | |
| 568 |
|
567 | |||
| 569 | if datatype==None: |
|
568 | if datatype==None: | |
| 570 | datatype = name.replace('Proc','') |
|
569 | datatype = name.replace('Proc','') | |
| 571 |
|
570 | |||
| 572 | self.id = str(id) |
|
571 | self.id = str(id) | |
| 573 | self.name = name |
|
572 | self.name = name | |
| 574 | self.datatype = datatype |
|
573 | self.datatype = datatype | |
| 575 | self.inputId = inputId |
|
574 | self.inputId = inputId | |
| 576 | self.parentId = parentId |
|
575 | self.parentId = parentId | |
| 577 |
|
576 | |||
| 578 | self.opConfObjList = [] |
|
577 | self.opConfObjList = [] | |
| 579 |
|
578 | |||
| 580 | self.addOperation(name='run', optype='self') |
|
579 | self.addOperation(name='run', optype='self') | |
| 581 |
|
580 | |||
| 582 | def removeOperations(self): |
|
581 | def removeOperations(self): | |
| 583 |
|
582 | |||
| 584 | for obj in self.opConfObjList: |
|
583 | for obj in self.opConfObjList: | |
| 585 | del obj |
|
584 | del obj | |
| 586 |
|
585 | |||
| 587 | self.opConfObjList = [] |
|
586 | self.opConfObjList = [] | |
| 588 | self.addOperation(name='run') |
|
587 | self.addOperation(name='run') | |
| 589 |
|
588 | |||
| 590 | def addParameter(self, **kwargs): |
|
589 | def addParameter(self, **kwargs): | |
| 591 | ''' |
|
590 | ''' | |
| 592 | Add parameters to "run" operation |
|
591 | Add parameters to "run" operation | |
| 593 | ''' |
|
592 | ''' | |
| 594 | opObj = self.opConfObjList[0] |
|
593 | opObj = self.opConfObjList[0] | |
| 595 |
|
594 | |||
| 596 | opObj.addParameter(**kwargs) |
|
595 | opObj.addParameter(**kwargs) | |
| 597 |
|
596 | |||
| 598 | return opObj |
|
597 | return opObj | |
| 599 |
|
598 | |||
| 600 | def addOperation(self, name, optype='self'): |
|
599 | def addOperation(self, name, optype='self'): | |
| 601 |
|
600 | |||
| 602 | id = self.__getNewId() |
|
601 | id = self.__getNewId() | |
| 603 | priority = self.__getPriority() |
|
602 | priority = self.__getPriority() | |
| 604 |
|
603 | |||
| 605 | opConfObj = OperationConf() |
|
604 | opConfObj = OperationConf() | |
| 606 | opConfObj.setup(id, name=name, priority=priority, type=optype) |
|
605 | opConfObj.setup(id, name=name, priority=priority, type=optype) | |
| 607 |
|
606 | |||
| 608 | self.opConfObjList.append(opConfObj) |
|
607 | self.opConfObjList.append(opConfObj) | |
| 609 |
|
608 | |||
| 610 | return opConfObj |
|
609 | return opConfObj | |
| 611 |
|
610 | |||
| 612 | def makeXml(self, projectElement): |
|
611 | def makeXml(self, projectElement): | |
| 613 |
|
612 | |||
| 614 | procUnitElement = SubElement(projectElement, self.ELEMENTNAME) |
|
613 | procUnitElement = SubElement(projectElement, self.ELEMENTNAME) | |
| 615 | procUnitElement.set('id', str(self.id)) |
|
614 | procUnitElement.set('id', str(self.id)) | |
| 616 | procUnitElement.set('name', self.name) |
|
615 | procUnitElement.set('name', self.name) | |
| 617 | procUnitElement.set('datatype', self.datatype) |
|
616 | procUnitElement.set('datatype', self.datatype) | |
| 618 | procUnitElement.set('inputId', str(self.inputId)) |
|
617 | procUnitElement.set('inputId', str(self.inputId)) | |
| 619 |
|
618 | |||
| 620 | for opConfObj in self.opConfObjList: |
|
619 | for opConfObj in self.opConfObjList: | |
| 621 | opConfObj.makeXml(procUnitElement) |
|
620 | opConfObj.makeXml(procUnitElement) | |
| 622 |
|
621 | |||
| 623 | def readXml(self, upElement): |
|
622 | def readXml(self, upElement): | |
| 624 |
|
623 | |||
| 625 | self.id = upElement.get('id') |
|
624 | self.id = upElement.get('id') | |
| 626 | self.name = upElement.get('name') |
|
625 | self.name = upElement.get('name') | |
| 627 | self.datatype = upElement.get('datatype') |
|
626 | self.datatype = upElement.get('datatype') | |
| 628 | self.inputId = upElement.get('inputId') |
|
627 | self.inputId = upElement.get('inputId') | |
| 629 |
|
628 | |||
| 630 | if self.ELEMENTNAME == "ReadUnit": |
|
629 | if self.ELEMENTNAME == "ReadUnit": | |
| 631 | self.datatype = self.datatype.replace("Reader", "") |
|
630 | self.datatype = self.datatype.replace("Reader", "") | |
| 632 |
|
631 | |||
| 633 | if self.ELEMENTNAME == "ProcUnit": |
|
632 | if self.ELEMENTNAME == "ProcUnit": | |
| 634 | self.datatype = self.datatype.replace("Proc", "") |
|
633 | self.datatype = self.datatype.replace("Proc", "") | |
| 635 |
|
634 | |||
| 636 | if self.inputId == 'None': |
|
635 | if self.inputId == 'None': | |
| 637 | self.inputId = '0' |
|
636 | self.inputId = '0' | |
| 638 |
|
637 | |||
| 639 | self.opConfObjList = [] |
|
638 | self.opConfObjList = [] | |
| 640 |
|
639 | |||
| 641 | opElementList = upElement.iter(OperationConf().getElementName()) |
|
640 | opElementList = upElement.iter(OperationConf().getElementName()) | |
| 642 |
|
641 | |||
| 643 | for opElement in opElementList: |
|
642 | for opElement in opElementList: | |
| 644 | opConfObj = OperationConf() |
|
643 | opConfObj = OperationConf() | |
| 645 | opConfObj.readXml(opElement) |
|
644 | opConfObj.readXml(opElement) | |
| 646 | self.opConfObjList.append(opConfObj) |
|
645 | self.opConfObjList.append(opConfObj) | |
| 647 |
|
646 | |||
| 648 | def printattr(self): |
|
647 | def printattr(self): | |
| 649 |
|
648 | |||
| 650 | print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME, |
|
649 | print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME, | |
| 651 | self.id, |
|
650 | self.id, | |
| 652 | self.name, |
|
651 | self.name, | |
| 653 | self.datatype, |
|
652 | self.datatype, | |
| 654 | self.inputId) |
|
653 | self.inputId) | |
| 655 |
|
654 | |||
| 656 | for opConfObj in self.opConfObjList: |
|
655 | for opConfObj in self.opConfObjList: | |
| 657 | opConfObj.printattr() |
|
656 | opConfObj.printattr() | |
| 658 |
|
657 | |||
| 659 |
|
658 | |||
| 660 | def getKwargs(self): |
|
659 | def getKwargs(self): | |
| 661 |
|
660 | |||
| 662 | opObj = self.opConfObjList[0] |
|
661 | opObj = self.opConfObjList[0] | |
| 663 | kwargs = opObj.getKwargs() |
|
662 | kwargs = opObj.getKwargs() | |
| 664 |
|
663 | |||
| 665 | return kwargs |
|
664 | return kwargs | |
| 666 |
|
665 | |||
| 667 | def createObjects(self, plotter_queue=None): |
|
666 | def createObjects(self, plotter_queue=None): | |
| 668 |
|
667 | |||
| 669 | className = eval(self.name) |
|
668 | className = eval(self.name) | |
| 670 | kwargs = self.getKwargs() |
|
669 | kwargs = self.getKwargs() | |
| 671 | procUnitObj = className(**kwargs) |
|
670 | procUnitObj = className(**kwargs) | |
| 672 |
|
671 | |||
| 673 | for opConfObj in self.opConfObjList: |
|
672 | for opConfObj in self.opConfObjList: | |
| 674 |
|
673 | |||
| 675 | if opConfObj.type=='self' and self.name=='run': |
|
674 | if opConfObj.type=='self' and self.name=='run': | |
| 676 | continue |
|
675 | continue | |
| 677 | elif opConfObj.type=='self': |
|
676 | elif opConfObj.type=='self': | |
| 678 | procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs()) |
|
677 | procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs()) | |
| 679 | continue |
|
678 | continue | |
| 680 |
|
679 | |||
| 681 | opObj = opConfObj.createObject(plotter_queue) |
|
680 | opObj = opConfObj.createObject(plotter_queue) | |
| 682 |
|
681 | |||
| 683 | self.opObjDict[opConfObj.id] = opObj |
|
682 | self.opObjDict[opConfObj.id] = opObj | |
| 684 |
|
683 | |||
| 685 | procUnitObj.addOperation(opObj, opConfObj.id) |
|
684 | procUnitObj.addOperation(opObj, opConfObj.id) | |
| 686 |
|
685 | |||
| 687 | self.procUnitObj = procUnitObj |
|
686 | self.procUnitObj = procUnitObj | |
| 688 |
|
687 | |||
| 689 | return procUnitObj |
|
688 | return procUnitObj | |
| 690 |
|
689 | |||
| 691 | def run(self): |
|
690 | def run(self): | |
| 692 |
|
691 | |||
| 693 | is_ok = False |
|
692 | is_ok = False | |
| 694 |
|
693 | |||
| 695 | for opConfObj in self.opConfObjList: |
|
694 | for opConfObj in self.opConfObjList: | |
| 696 |
|
695 | |||
| 697 | kwargs = {} |
|
696 | kwargs = {} | |
| 698 | for parmConfObj in opConfObj.getParameterObjList(): |
|
697 | for parmConfObj in opConfObj.getParameterObjList(): | |
| 699 | if opConfObj.name == 'run' and parmConfObj.name == 'datatype': |
|
698 | if opConfObj.name == 'run' and parmConfObj.name == 'datatype': | |
| 700 | continue |
|
699 | continue | |
| 701 |
|
700 | |||
| 702 | kwargs[parmConfObj.name] = parmConfObj.getValue() |
|
701 | kwargs[parmConfObj.name] = parmConfObj.getValue() | |
| 703 |
|
702 | |||
| 704 | #ini = time.time() |
|
703 | #ini = time.time() | |
| 705 |
|
704 | |||
| 706 | #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id) |
|
705 | #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id) | |
| 707 | sts = self.procUnitObj.call(opType = opConfObj.type, |
|
706 | sts = self.procUnitObj.call(opType = opConfObj.type, | |
| 708 | opName = opConfObj.name, |
|
707 | opName = opConfObj.name, | |
| 709 | opId = opConfObj.id, |
|
708 | opId = opConfObj.id, | |
| 710 | ) |
|
709 | ) | |
| 711 |
|
710 | |||
| 712 | # total_time = time.time() - ini |
|
711 | # total_time = time.time() - ini | |
| 713 | # |
|
712 | # | |
| 714 | # if total_time > 0.002: |
|
713 | # if total_time > 0.002: | |
| 715 | # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time) |
|
714 | # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time) | |
| 716 |
|
715 | |||
| 717 | is_ok = is_ok or sts |
|
716 | is_ok = is_ok or sts | |
| 718 |
|
717 | |||
| 719 | return is_ok |
|
718 | return is_ok | |
| 720 |
|
719 | |||
| 721 | def close(self): |
|
720 | def close(self): | |
| 722 |
|
721 | |||
| 723 | for opConfObj in self.opConfObjList: |
|
722 | for opConfObj in self.opConfObjList: | |
| 724 | if opConfObj.type == 'self': |
|
723 | if opConfObj.type == 'self': | |
| 725 | continue |
|
724 | continue | |
| 726 |
|
725 | |||
| 727 | opObj = self.procUnitObj.getOperationObj(opConfObj.id) |
|
726 | opObj = self.procUnitObj.getOperationObj(opConfObj.id) | |
| 728 | opObj.close() |
|
727 | opObj.close() | |
| 729 |
|
728 | |||
| 730 | self.procUnitObj.close() |
|
729 | self.procUnitObj.close() | |
| 731 |
|
730 | |||
| 732 | return |
|
731 | return | |
| 733 |
|
732 | |||
| 734 | class ReadUnitConf(ProcUnitConf): |
|
733 | class ReadUnitConf(ProcUnitConf): | |
| 735 |
|
734 | |||
| 736 | path = None |
|
735 | path = None | |
| 737 | startDate = None |
|
736 | startDate = None | |
| 738 | endDate = None |
|
737 | endDate = None | |
| 739 | startTime = None |
|
738 | startTime = None | |
| 740 | endTime = None |
|
739 | endTime = None | |
| 741 |
|
740 | |||
| 742 | ELEMENTNAME = 'ReadUnit' |
|
741 | ELEMENTNAME = 'ReadUnit' | |
| 743 |
|
742 | |||
| 744 | def __init__(self): |
|
743 | def __init__(self): | |
| 745 |
|
744 | |||
| 746 | self.id = None |
|
745 | self.id = None | |
| 747 | self.datatype = None |
|
746 | self.datatype = None | |
| 748 | self.name = None |
|
747 | self.name = None | |
| 749 | self.inputId = None |
|
748 | self.inputId = None | |
| 750 |
|
749 | |||
| 751 | self.parentId = None |
|
750 | self.parentId = None | |
| 752 |
|
751 | |||
| 753 | self.opConfObjList = [] |
|
752 | self.opConfObjList = [] | |
| 754 | self.opObjList = [] |
|
753 | self.opObjList = [] | |
| 755 |
|
754 | |||
| 756 | def getElementName(self): |
|
755 | def getElementName(self): | |
| 757 |
|
756 | |||
| 758 | return self.ELEMENTNAME |
|
757 | return self.ELEMENTNAME | |
| 759 |
|
758 | |||
| 760 |
def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", |
|
759 | def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="", | |
|
|
760 | endTime="", parentId=None, queue=None, server=None, **kwargs): | |||
| 761 |
|
761 | |||
| 762 | #Compatible with old signal chain version |
|
762 | #Compatible with old signal chain version | |
| 763 | if datatype==None and name==None: |
|
763 | if datatype==None and name==None: | |
| 764 | raise ValueError, "datatype or name should be defined" |
|
764 | raise ValueError, "datatype or name should be defined" | |
| 765 |
|
765 | |||
| 766 | if name==None: |
|
766 | if name==None: | |
| 767 | if 'Reader' in datatype: |
|
767 | if 'Reader' in datatype: | |
| 768 | name = datatype |
|
768 | name = datatype | |
| 769 | else: |
|
769 | else: | |
| 770 | name = '%sReader' %(datatype) |
|
770 | name = '%sReader' %(datatype) | |
| 771 |
|
||||
| 772 | if datatype==None: |
|
771 | if datatype==None: | |
| 773 | datatype = name.replace('Reader','') |
|
772 | datatype = name.replace('Reader','') | |
| 774 |
|
773 | |||
| 775 | self.id = id |
|
774 | self.id = id | |
| 776 | self.name = name |
|
775 | self.name = name | |
| 777 | self.datatype = datatype |
|
776 | self.datatype = datatype | |
| 778 |
|
777 | if path != '': | ||
| 779 | self.path = os.path.abspath(path) |
|
778 | self.path = os.path.abspath(path) | |
| 780 | self.startDate = startDate |
|
779 | self.startDate = startDate | |
| 781 | self.endDate = endDate |
|
780 | self.endDate = endDate | |
| 782 | self.startTime = startTime |
|
781 | self.startTime = startTime | |
| 783 | self.endTime = endTime |
|
782 | self.endTime = endTime | |
| 784 |
|
783 | |||
| 785 | self.inputId = '0' |
|
784 | self.inputId = '0' | |
| 786 | self.parentId = parentId |
|
785 | self.parentId = parentId | |
| 787 | self.queue = queue |
|
786 | self.queue = queue | |
|
|
787 | self.server = server | |||
| 788 | self.addRunOperation(**kwargs) |
|
788 | self.addRunOperation(**kwargs) | |
| 789 |
|
789 | |||
| 790 | def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs): |
|
790 | def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs): | |
| 791 |
|
791 | |||
| 792 | #Compatible with old signal chain version |
|
792 | #Compatible with old signal chain version | |
| 793 | if datatype==None and name==None: |
|
793 | if datatype==None and name==None: | |
| 794 | raise ValueError, "datatype or name should be defined" |
|
794 | raise ValueError, "datatype or name should be defined" | |
| 795 |
|
795 | |||
| 796 | if name==None: |
|
796 | if name==None: | |
| 797 | if 'Reader' in datatype: |
|
797 | if 'Reader' in datatype: | |
| 798 | name = datatype |
|
798 | name = datatype | |
| 799 | else: |
|
799 | else: | |
| 800 | name = '%sReader' %(datatype) |
|
800 | name = '%sReader' %(datatype) | |
| 801 |
|
801 | |||
| 802 | if datatype==None: |
|
802 | if datatype==None: | |
| 803 | datatype = name.replace('Reader','') |
|
803 | datatype = name.replace('Reader','') | |
| 804 |
|
804 | |||
| 805 | self.datatype = datatype |
|
805 | self.datatype = datatype | |
| 806 | self.name = name |
|
806 | self.name = name | |
| 807 | self.path = path |
|
807 | self.path = path | |
| 808 | self.startDate = startDate |
|
808 | self.startDate = startDate | |
| 809 | self.endDate = endDate |
|
809 | self.endDate = endDate | |
| 810 | self.startTime = startTime |
|
810 | self.startTime = startTime | |
| 811 | self.endTime = endTime |
|
811 | self.endTime = endTime | |
| 812 |
|
812 | |||
| 813 | self.inputId = '0' |
|
813 | self.inputId = '0' | |
| 814 | self.parentId = parentId |
|
814 | self.parentId = parentId | |
| 815 |
|
815 | |||
| 816 | self.updateRunOperation(**kwargs) |
|
816 | self.updateRunOperation(**kwargs) | |
| 817 |
|
817 | |||
| 818 | def removeOperations(self): |
|
818 | def removeOperations(self): | |
| 819 |
|
819 | |||
| 820 | for obj in self.opConfObjList: |
|
820 | for obj in self.opConfObjList: | |
| 821 | del obj |
|
821 | del obj | |
| 822 |
|
822 | |||
| 823 | self.opConfObjList = [] |
|
823 | self.opConfObjList = [] | |
| 824 |
|
824 | |||
| 825 | def addRunOperation(self, **kwargs): |
|
825 | def addRunOperation(self, **kwargs): | |
| 826 |
|
826 | |||
| 827 | opObj = self.addOperation(name = 'run', optype = 'self') |
|
827 | opObj = self.addOperation(name = 'run', optype = 'self') | |
| 828 |
|
828 | |||
| 829 | opObj.addParameter(name='datatype' , value=self.datatype, format='str') |
|
829 | if self.server is None: | |
| 830 |
opObj.addParameter(name=' |
|
830 | opObj.addParameter(name='datatype' , value=self.datatype, format='str') | |
| 831 |
opObj.addParameter(name=' |
|
831 | opObj.addParameter(name='path' , value=self.path, format='str') | |
| 832 |
opObj.addParameter(name=' |
|
832 | opObj.addParameter(name='startDate' , value=self.startDate, format='date') | |
| 833 |
opObj.addParameter(name=' |
|
833 | opObj.addParameter(name='endDate' , value=self.endDate, format='date') | |
| 834 |
opObj.addParameter(name=' |
|
834 | opObj.addParameter(name='startTime' , value=self.startTime, format='time') | |
| 835 |
opObj.addParameter(name=' |
|
835 | opObj.addParameter(name='endTime' , value=self.endTime, format='time') | |
| 836 |
|
836 | opObj.addParameter(name='queue' , value=self.queue, format='obj') | ||
| 837 | for key, value in kwargs.items(): |
|
837 | for key, value in kwargs.items(): | |
| 838 | opObj.addParameter(name=key, value=value, format=type(value).__name__) |
|
838 | opObj.addParameter(name=key, value=value, format=type(value).__name__) | |
|
|
839 | else: | |||
|
|
840 | opObj.addParameter(name='server' , value=self.server, format='str') | |||
|
|
841 | ||||
| 839 |
|
842 | |||
| 840 | return opObj |
|
843 | return opObj | |
| 841 |
|
844 | |||
| 842 | def updateRunOperation(self, **kwargs): |
|
845 | def updateRunOperation(self, **kwargs): | |
| 843 |
|
846 | |||
| 844 | opObj = self.getOperationObj(name = 'run') |
|
847 | opObj = self.getOperationObj(name = 'run') | |
| 845 | opObj.removeParameters() |
|
848 | opObj.removeParameters() | |
| 846 |
|
849 | |||
| 847 | opObj.addParameter(name='datatype' , value=self.datatype, format='str') |
|
850 | opObj.addParameter(name='datatype' , value=self.datatype, format='str') | |
| 848 | opObj.addParameter(name='path' , value=self.path, format='str') |
|
851 | opObj.addParameter(name='path' , value=self.path, format='str') | |
| 849 | opObj.addParameter(name='startDate' , value=self.startDate, format='date') |
|
852 | opObj.addParameter(name='startDate' , value=self.startDate, format='date') | |
| 850 | opObj.addParameter(name='endDate' , value=self.endDate, format='date') |
|
853 | opObj.addParameter(name='endDate' , value=self.endDate, format='date') | |
| 851 | opObj.addParameter(name='startTime' , value=self.startTime, format='time') |
|
854 | opObj.addParameter(name='startTime' , value=self.startTime, format='time') | |
| 852 | opObj.addParameter(name='endTime' , value=self.endTime, format='time') |
|
855 | opObj.addParameter(name='endTime' , value=self.endTime, format='time') | |
| 853 |
|
856 | |||
| 854 | for key, value in kwargs.items(): |
|
857 | for key, value in kwargs.items(): | |
| 855 | opObj.addParameter(name=key, value=value, format=type(value).__name__) |
|
858 | opObj.addParameter(name=key, value=value, format=type(value).__name__) | |
| 856 |
|
859 | |||
| 857 | return opObj |
|
860 | return opObj | |
| 858 |
|
861 | |||
| 859 | # def makeXml(self, projectElement): |
|
862 | # def makeXml(self, projectElement): | |
| 860 | # |
|
863 | # | |
| 861 | # procUnitElement = SubElement(projectElement, self.ELEMENTNAME) |
|
864 | # procUnitElement = SubElement(projectElement, self.ELEMENTNAME) | |
| 862 | # procUnitElement.set('id', str(self.id)) |
|
865 | # procUnitElement.set('id', str(self.id)) | |
| 863 | # procUnitElement.set('name', self.name) |
|
866 | # procUnitElement.set('name', self.name) | |
| 864 | # procUnitElement.set('datatype', self.datatype) |
|
867 | # procUnitElement.set('datatype', self.datatype) | |
| 865 | # procUnitElement.set('inputId', str(self.inputId)) |
|
868 | # procUnitElement.set('inputId', str(self.inputId)) | |
| 866 | # |
|
869 | # | |
| 867 | # for opConfObj in self.opConfObjList: |
|
870 | # for opConfObj in self.opConfObjList: | |
| 868 | # opConfObj.makeXml(procUnitElement) |
|
871 | # opConfObj.makeXml(procUnitElement) | |
| 869 |
|
872 | |||
| 870 | def readXml(self, upElement): |
|
873 | def readXml(self, upElement): | |
| 871 |
|
874 | |||
| 872 | self.id = upElement.get('id') |
|
875 | self.id = upElement.get('id') | |
| 873 | self.name = upElement.get('name') |
|
876 | self.name = upElement.get('name') | |
| 874 | self.datatype = upElement.get('datatype') |
|
877 | self.datatype = upElement.get('datatype') | |
| 875 | self.inputId = upElement.get('inputId') |
|
878 | self.inputId = upElement.get('inputId') | |
| 876 |
|
879 | |||
| 877 | if self.ELEMENTNAME == "ReadUnit": |
|
880 | if self.ELEMENTNAME == "ReadUnit": | |
| 878 | self.datatype = self.datatype.replace("Reader", "") |
|
881 | self.datatype = self.datatype.replace("Reader", "") | |
| 879 |
|
882 | |||
| 880 | if self.inputId == 'None': |
|
883 | if self.inputId == 'None': | |
| 881 | self.inputId = '0' |
|
884 | self.inputId = '0' | |
| 882 |
|
885 | |||
| 883 | self.opConfObjList = [] |
|
886 | self.opConfObjList = [] | |
| 884 |
|
887 | |||
| 885 | opElementList = upElement.iter(OperationConf().getElementName()) |
|
888 | opElementList = upElement.iter(OperationConf().getElementName()) | |
| 886 |
|
889 | |||
| 887 | for opElement in opElementList: |
|
890 | for opElement in opElementList: | |
| 888 | opConfObj = OperationConf() |
|
891 | opConfObj = OperationConf() | |
| 889 | opConfObj.readXml(opElement) |
|
892 | opConfObj.readXml(opElement) | |
| 890 | self.opConfObjList.append(opConfObj) |
|
893 | self.opConfObjList.append(opConfObj) | |
| 891 |
|
894 | |||
| 892 | if opConfObj.name == 'run': |
|
895 | if opConfObj.name == 'run': | |
| 893 | self.path = opConfObj.getParameterValue('path') |
|
896 | self.path = opConfObj.getParameterValue('path') | |
| 894 | self.startDate = opConfObj.getParameterValue('startDate') |
|
897 | self.startDate = opConfObj.getParameterValue('startDate') | |
| 895 | self.endDate = opConfObj.getParameterValue('endDate') |
|
898 | self.endDate = opConfObj.getParameterValue('endDate') | |
| 896 | self.startTime = opConfObj.getParameterValue('startTime') |
|
899 | self.startTime = opConfObj.getParameterValue('startTime') | |
| 897 | self.endTime = opConfObj.getParameterValue('endTime') |
|
900 | self.endTime = opConfObj.getParameterValue('endTime') | |
| 898 |
|
901 | |||
| 899 | class Project(): |
|
902 | class Project(): | |
| 900 |
|
903 | |||
| 901 | id = None |
|
904 | id = None | |
| 902 | name = None |
|
905 | name = None | |
| 903 | description = None |
|
906 | description = None | |
| 904 | filename = None |
|
907 | filename = None | |
| 905 |
|
908 | |||
| 906 | procUnitConfObjDict = None |
|
909 | procUnitConfObjDict = None | |
| 907 |
|
910 | |||
| 908 | ELEMENTNAME = 'Project' |
|
911 | ELEMENTNAME = 'Project' | |
| 909 |
|
912 | |||
| 910 | plotterQueue = None |
|
913 | plotterQueue = None | |
| 911 |
|
914 | |||
| 912 | def __init__(self, plotter_queue=None): |
|
915 | def __init__(self, plotter_queue=None): | |
| 913 |
|
916 | |||
| 914 | self.id = None |
|
917 | self.id = None | |
| 915 | self.name = None |
|
918 | self.name = None | |
| 916 | self.description = None |
|
919 | self.description = None | |
| 917 |
|
920 | |||
| 918 | self.plotterQueue = plotter_queue |
|
921 | self.plotterQueue = plotter_queue | |
| 919 |
|
922 | |||
| 920 | self.procUnitConfObjDict = {} |
|
923 | self.procUnitConfObjDict = {} | |
| 921 |
|
924 | |||
| 922 | def __getNewId(self): |
|
925 | def __getNewId(self): | |
| 923 |
|
926 | |||
| 924 | idList = self.procUnitConfObjDict.keys() |
|
927 | idList = self.procUnitConfObjDict.keys() | |
| 925 |
|
928 | |||
| 926 | id = int(self.id)*10 |
|
929 | id = int(self.id)*10 | |
| 927 |
|
930 | |||
| 928 | while True: |
|
931 | while True: | |
| 929 | id += 1 |
|
932 | id += 1 | |
| 930 |
|
933 | |||
| 931 | if str(id) in idList: |
|
934 | if str(id) in idList: | |
| 932 | continue |
|
935 | continue | |
| 933 |
|
936 | |||
| 934 | break |
|
937 | break | |
| 935 |
|
938 | |||
| 936 | return str(id) |
|
939 | return str(id) | |
| 937 |
|
940 | |||
| 938 | def getElementName(self): |
|
941 | def getElementName(self): | |
| 939 |
|
942 | |||
| 940 | return self.ELEMENTNAME |
|
943 | return self.ELEMENTNAME | |
| 941 |
|
944 | |||
| 942 | def getId(self): |
|
945 | def getId(self): | |
| 943 |
|
946 | |||
| 944 | return self.id |
|
947 | return self.id | |
| 945 |
|
948 | |||
| 946 | def updateId(self, new_id): |
|
949 | def updateId(self, new_id): | |
| 947 |
|
950 | |||
| 948 | self.id = str(new_id) |
|
951 | self.id = str(new_id) | |
| 949 |
|
952 | |||
| 950 | keyList = self.procUnitConfObjDict.keys() |
|
953 | keyList = self.procUnitConfObjDict.keys() | |
| 951 | keyList.sort() |
|
954 | keyList.sort() | |
| 952 |
|
955 | |||
| 953 | n = 1 |
|
956 | n = 1 | |
| 954 | newProcUnitConfObjDict = {} |
|
957 | newProcUnitConfObjDict = {} | |
| 955 |
|
958 | |||
| 956 | for procKey in keyList: |
|
959 | for procKey in keyList: | |
| 957 |
|
960 | |||
| 958 | procUnitConfObj = self.procUnitConfObjDict[procKey] |
|
961 | procUnitConfObj = self.procUnitConfObjDict[procKey] | |
| 959 | idProcUnit = str(int(self.id)*10 + n) |
|
962 | idProcUnit = str(int(self.id)*10 + n) | |
| 960 | procUnitConfObj.updateId(idProcUnit, parentId = self.id) |
|
963 | procUnitConfObj.updateId(idProcUnit, parentId = self.id) | |
| 961 |
|
964 | |||
| 962 | newProcUnitConfObjDict[idProcUnit] = procUnitConfObj |
|
965 | newProcUnitConfObjDict[idProcUnit] = procUnitConfObj | |
| 963 | n += 1 |
|
966 | n += 1 | |
| 964 |
|
967 | |||
| 965 | self.procUnitConfObjDict = newProcUnitConfObjDict |
|
968 | self.procUnitConfObjDict = newProcUnitConfObjDict | |
| 966 |
|
969 | |||
| 967 | def setup(self, id, name, description): |
|
970 | def setup(self, id, name, description): | |
| 968 |
|
971 | |||
| 969 | self.id = str(id) |
|
972 | self.id = str(id) | |
| 970 | self.name = name |
|
973 | self.name = name | |
| 971 | self.description = description |
|
974 | self.description = description | |
| 972 |
|
975 | |||
| 973 | def update(self, name, description): |
|
976 | def update(self, name, description): | |
| 974 |
|
977 | |||
| 975 | self.name = name |
|
978 | self.name = name | |
| 976 | self.description = description |
|
979 | self.description = description | |
| 977 |
|
980 | |||
| 978 | def addReadUnit(self, id=None, datatype=None, name=None, **kwargs): |
|
981 | def addReadUnit(self, id=None, datatype=None, name=None, **kwargs): | |
| 979 |
|
982 | |||
| 980 | if id is None: |
|
983 | if id is None: | |
| 981 | idReadUnit = self.__getNewId() |
|
984 | idReadUnit = self.__getNewId() | |
| 982 | else: |
|
985 | else: | |
| 983 | idReadUnit = str(id) |
|
986 | idReadUnit = str(id) | |
| 984 |
|
987 | |||
| 985 | readUnitConfObj = ReadUnitConf() |
|
988 | readUnitConfObj = ReadUnitConf() | |
| 986 | readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs) |
|
989 | readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs) | |
| 987 |
|
990 | |||
| 988 | self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj |
|
991 | self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj | |
| 989 |
|
992 | |||
| 990 | return readUnitConfObj |
|
993 | return readUnitConfObj | |
| 991 |
|
994 | |||
| 992 | def addProcUnit(self, inputId='0', datatype=None, name=None): |
|
995 | def addProcUnit(self, inputId='0', datatype=None, name=None): | |
| 993 |
|
996 | |||
| 994 | idProcUnit = self.__getNewId() |
|
997 | idProcUnit = self.__getNewId() | |
| 995 |
|
998 | |||
| 996 | procUnitConfObj = ProcUnitConf() |
|
999 | procUnitConfObj = ProcUnitConf() | |
| 997 | procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id) |
|
1000 | procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id) | |
| 998 |
|
1001 | |||
| 999 | self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj |
|
1002 | self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj | |
| 1000 |
|
1003 | |||
| 1001 | return procUnitConfObj |
|
1004 | return procUnitConfObj | |
| 1002 |
|
1005 | |||
| 1003 | def removeProcUnit(self, id): |
|
1006 | def removeProcUnit(self, id): | |
| 1004 |
|
1007 | |||
| 1005 | if id in self.procUnitConfObjDict.keys(): |
|
1008 | if id in self.procUnitConfObjDict.keys(): | |
| 1006 | self.procUnitConfObjDict.pop(id) |
|
1009 | self.procUnitConfObjDict.pop(id) | |
| 1007 |
|
1010 | |||
| 1008 | def getReadUnitId(self): |
|
1011 | def getReadUnitId(self): | |
| 1009 |
|
1012 | |||
| 1010 | readUnitConfObj = self.getReadUnitObj() |
|
1013 | readUnitConfObj = self.getReadUnitObj() | |
| 1011 |
|
1014 | |||
| 1012 | return readUnitConfObj.id |
|
1015 | return readUnitConfObj.id | |
| 1013 |
|
1016 | |||
| 1014 | def getReadUnitObj(self): |
|
1017 | def getReadUnitObj(self): | |
| 1015 |
|
1018 | |||
| 1016 | for obj in self.procUnitConfObjDict.values(): |
|
1019 | for obj in self.procUnitConfObjDict.values(): | |
| 1017 | if obj.getElementName() == "ReadUnit": |
|
1020 | if obj.getElementName() == "ReadUnit": | |
| 1018 | return obj |
|
1021 | return obj | |
| 1019 |
|
1022 | |||
| 1020 | return None |
|
1023 | return None | |
| 1021 |
|
1024 | |||
| 1022 | def getProcUnitObj(self, id=None, name=None): |
|
1025 | def getProcUnitObj(self, id=None, name=None): | |
| 1023 |
|
1026 | |||
| 1024 | if id != None: |
|
1027 | if id != None: | |
| 1025 | return self.procUnitConfObjDict[id] |
|
1028 | return self.procUnitConfObjDict[id] | |
| 1026 |
|
1029 | |||
| 1027 | if name != None: |
|
1030 | if name != None: | |
| 1028 | return self.getProcUnitObjByName(name) |
|
1031 | return self.getProcUnitObjByName(name) | |
| 1029 |
|
1032 | |||
| 1030 | return None |
|
1033 | return None | |
| 1031 |
|
1034 | |||
| 1032 | def getProcUnitObjByName(self, name): |
|
1035 | def getProcUnitObjByName(self, name): | |
| 1033 |
|
1036 | |||
| 1034 | for obj in self.procUnitConfObjDict.values(): |
|
1037 | for obj in self.procUnitConfObjDict.values(): | |
| 1035 | if obj.name == name: |
|
1038 | if obj.name == name: | |
| 1036 | return obj |
|
1039 | return obj | |
| 1037 |
|
1040 | |||
| 1038 | return None |
|
1041 | return None | |
| 1039 |
|
1042 | |||
| 1040 | def procUnitItems(self): |
|
1043 | def procUnitItems(self): | |
| 1041 |
|
1044 | |||
| 1042 | return self.procUnitConfObjDict.items() |
|
1045 | return self.procUnitConfObjDict.items() | |
| 1043 |
|
1046 | |||
| 1044 | def makeXml(self): |
|
1047 | def makeXml(self): | |
| 1045 |
|
1048 | |||
| 1046 | projectElement = Element('Project') |
|
1049 | projectElement = Element('Project') | |
| 1047 | projectElement.set('id', str(self.id)) |
|
1050 | projectElement.set('id', str(self.id)) | |
| 1048 | projectElement.set('name', self.name) |
|
1051 | projectElement.set('name', self.name) | |
| 1049 | projectElement.set('description', self.description) |
|
1052 | projectElement.set('description', self.description) | |
| 1050 |
|
1053 | |||
| 1051 | for procUnitConfObj in self.procUnitConfObjDict.values(): |
|
1054 | for procUnitConfObj in self.procUnitConfObjDict.values(): | |
| 1052 | procUnitConfObj.makeXml(projectElement) |
|
1055 | procUnitConfObj.makeXml(projectElement) | |
| 1053 |
|
1056 | |||
| 1054 | self.projectElement = projectElement |
|
1057 | self.projectElement = projectElement | |
| 1055 |
|
1058 | |||
| 1056 | def writeXml(self, filename=None): |
|
1059 | def writeXml(self, filename=None): | |
| 1057 |
|
1060 | |||
| 1058 | if filename == None: |
|
1061 | if filename == None: | |
| 1059 | if self.filename: |
|
1062 | if self.filename: | |
| 1060 | filename = self.filename |
|
1063 | filename = self.filename | |
| 1061 | else: |
|
1064 | else: | |
| 1062 | filename = "schain.xml" |
|
1065 | filename = "schain.xml" | |
| 1063 |
|
1066 | |||
| 1064 | if not filename: |
|
1067 | if not filename: | |
| 1065 | print "filename has not been defined. Use setFilename(filename) for do it." |
|
1068 | print "filename has not been defined. Use setFilename(filename) for do it." | |
| 1066 | return 0 |
|
1069 | return 0 | |
| 1067 |
|
1070 | |||
| 1068 | abs_file = os.path.abspath(filename) |
|
1071 | abs_file = os.path.abspath(filename) | |
| 1069 |
|
1072 | |||
| 1070 | if not os.access(os.path.dirname(abs_file), os.W_OK): |
|
1073 | if not os.access(os.path.dirname(abs_file), os.W_OK): | |
| 1071 | print "No write permission on %s" %os.path.dirname(abs_file) |
|
1074 | print "No write permission on %s" %os.path.dirname(abs_file) | |
| 1072 | return 0 |
|
1075 | return 0 | |
| 1073 |
|
1076 | |||
| 1074 | if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)): |
|
1077 | if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)): | |
| 1075 | print "File %s already exists and it could not be overwriten" %abs_file |
|
1078 | print "File %s already exists and it could not be overwriten" %abs_file | |
| 1076 | return 0 |
|
1079 | return 0 | |
| 1077 |
|
1080 | |||
| 1078 | self.makeXml() |
|
1081 | self.makeXml() | |
| 1079 |
|
1082 | |||
| 1080 | ElementTree(self.projectElement).write(abs_file, method='xml') |
|
1083 | ElementTree(self.projectElement).write(abs_file, method='xml') | |
| 1081 |
|
1084 | |||
| 1082 | self.filename = abs_file |
|
1085 | self.filename = abs_file | |
| 1083 |
|
1086 | |||
| 1084 | return 1 |
|
1087 | return 1 | |
| 1085 |
|
1088 | |||
| 1086 | def readXml(self, filename = None): |
|
1089 | def readXml(self, filename = None): | |
| 1087 |
|
1090 | |||
| 1088 | if not filename: |
|
1091 | if not filename: | |
| 1089 | print "filename is not defined" |
|
1092 | print "filename is not defined" | |
| 1090 | return 0 |
|
1093 | return 0 | |
| 1091 |
|
1094 | |||
| 1092 | abs_file = os.path.abspath(filename) |
|
1095 | abs_file = os.path.abspath(filename) | |
| 1093 |
|
1096 | |||
| 1094 | if not os.path.isfile(abs_file): |
|
1097 | if not os.path.isfile(abs_file): | |
| 1095 | print "%s file does not exist" %abs_file |
|
1098 | print "%s file does not exist" %abs_file | |
| 1096 | return 0 |
|
1099 | return 0 | |
| 1097 |
|
1100 | |||
| 1098 | self.projectElement = None |
|
1101 | self.projectElement = None | |
| 1099 | self.procUnitConfObjDict = {} |
|
1102 | self.procUnitConfObjDict = {} | |
| 1100 |
|
1103 | |||
| 1101 | try: |
|
1104 | try: | |
| 1102 | self.projectElement = ElementTree().parse(abs_file) |
|
1105 | self.projectElement = ElementTree().parse(abs_file) | |
| 1103 | except: |
|
1106 | except: | |
| 1104 | print "Error reading %s, verify file format" %filename |
|
1107 | print "Error reading %s, verify file format" %filename | |
| 1105 | return 0 |
|
1108 | return 0 | |
| 1106 |
|
1109 | |||
| 1107 | self.project = self.projectElement.tag |
|
1110 | self.project = self.projectElement.tag | |
| 1108 |
|
1111 | |||
| 1109 | self.id = self.projectElement.get('id') |
|
1112 | self.id = self.projectElement.get('id') | |
| 1110 | self.name = self.projectElement.get('name') |
|
1113 | self.name = self.projectElement.get('name') | |
| 1111 | self.description = self.projectElement.get('description') |
|
1114 | self.description = self.projectElement.get('description') | |
| 1112 |
|
1115 | |||
| 1113 | readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName()) |
|
1116 | readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName()) | |
| 1114 |
|
1117 | |||
| 1115 | for readUnitElement in readUnitElementList: |
|
1118 | for readUnitElement in readUnitElementList: | |
| 1116 | readUnitConfObj = ReadUnitConf() |
|
1119 | readUnitConfObj = ReadUnitConf() | |
| 1117 | readUnitConfObj.readXml(readUnitElement) |
|
1120 | readUnitConfObj.readXml(readUnitElement) | |
| 1118 |
|
1121 | |||
| 1119 | if readUnitConfObj.parentId == None: |
|
1122 | if readUnitConfObj.parentId == None: | |
| 1120 | readUnitConfObj.parentId = self.id |
|
1123 | readUnitConfObj.parentId = self.id | |
| 1121 |
|
1124 | |||
| 1122 | self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj |
|
1125 | self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj | |
| 1123 |
|
1126 | |||
| 1124 | procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName()) |
|
1127 | procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName()) | |
| 1125 |
|
1128 | |||
| 1126 | for procUnitElement in procUnitElementList: |
|
1129 | for procUnitElement in procUnitElementList: | |
| 1127 | procUnitConfObj = ProcUnitConf() |
|
1130 | procUnitConfObj = ProcUnitConf() | |
| 1128 | procUnitConfObj.readXml(procUnitElement) |
|
1131 | procUnitConfObj.readXml(procUnitElement) | |
| 1129 |
|
1132 | |||
| 1130 | if procUnitConfObj.parentId == None: |
|
1133 | if procUnitConfObj.parentId == None: | |
| 1131 | procUnitConfObj.parentId = self.id |
|
1134 | procUnitConfObj.parentId = self.id | |
| 1132 |
|
1135 | |||
| 1133 | self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj |
|
1136 | self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj | |
| 1134 |
|
1137 | |||
| 1135 | self.filename = abs_file |
|
1138 | self.filename = abs_file | |
| 1136 |
|
1139 | |||
| 1137 | return 1 |
|
1140 | return 1 | |
| 1138 |
|
1141 | |||
| 1139 | def printattr(self): |
|
1142 | def printattr(self): | |
| 1140 |
|
1143 | |||
| 1141 | print "Project[%s]: name = %s, description = %s" %(self.id, |
|
1144 | print "Project[%s]: name = %s, description = %s" %(self.id, | |
| 1142 | self.name, |
|
1145 | self.name, | |
| 1143 | self.description) |
|
1146 | self.description) | |
| 1144 |
|
1147 | |||
| 1145 | for procUnitConfObj in self.procUnitConfObjDict.values(): |
|
1148 | for procUnitConfObj in self.procUnitConfObjDict.values(): | |
| 1146 | procUnitConfObj.printattr() |
|
1149 | procUnitConfObj.printattr() | |
| 1147 |
|
1150 | |||
| 1148 | def createObjects(self): |
|
1151 | def createObjects(self): | |
| 1149 |
|
1152 | |||
| 1150 | for procUnitConfObj in self.procUnitConfObjDict.values(): |
|
1153 | for procUnitConfObj in self.procUnitConfObjDict.values(): | |
| 1151 | procUnitConfObj.createObjects(self.plotterQueue) |
|
1154 | procUnitConfObj.createObjects(self.plotterQueue) | |
| 1152 |
|
1155 | |||
| 1153 | def __connect(self, objIN, thisObj): |
|
1156 | def __connect(self, objIN, thisObj): | |
| 1154 |
|
1157 | |||
| 1155 | thisObj.setInput(objIN.getOutputObj()) |
|
1158 | thisObj.setInput(objIN.getOutputObj()) | |
| 1156 |
|
1159 | |||
| 1157 | def connectObjects(self): |
|
1160 | def connectObjects(self): | |
| 1158 |
|
1161 | |||
| 1159 | for thisPUConfObj in self.procUnitConfObjDict.values(): |
|
1162 | for thisPUConfObj in self.procUnitConfObjDict.values(): | |
| 1160 |
|
1163 | |||
| 1161 | inputId = thisPUConfObj.getInputId() |
|
1164 | inputId = thisPUConfObj.getInputId() | |
| 1162 |
|
1165 | |||
| 1163 | if int(inputId) == 0: |
|
1166 | if int(inputId) == 0: | |
| 1164 | continue |
|
1167 | continue | |
| 1165 |
|
1168 | |||
| 1166 | #Get input object |
|
1169 | #Get input object | |
| 1167 | puConfINObj = self.procUnitConfObjDict[inputId] |
|
1170 | puConfINObj = self.procUnitConfObjDict[inputId] | |
| 1168 | puObjIN = puConfINObj.getProcUnitObj() |
|
1171 | puObjIN = puConfINObj.getProcUnitObj() | |
| 1169 |
|
1172 | |||
| 1170 | #Get current object |
|
1173 | #Get current object | |
| 1171 | thisPUObj = thisPUConfObj.getProcUnitObj() |
|
1174 | thisPUObj = thisPUConfObj.getProcUnitObj() | |
| 1172 |
|
1175 | |||
| 1173 | self.__connect(puObjIN, thisPUObj) |
|
1176 | self.__connect(puObjIN, thisPUObj) | |
| 1174 |
|
1177 | |||
| 1175 | def __handleError(self, procUnitConfObj, send_email=True): |
|
1178 | def __handleError(self, procUnitConfObj, send_email=True): | |
| 1176 |
|
1179 | |||
| 1177 | import socket |
|
1180 | import socket | |
| 1178 |
|
1181 | |||
| 1179 | err = traceback.format_exception(sys.exc_info()[0], |
|
1182 | err = traceback.format_exception(sys.exc_info()[0], | |
| 1180 | sys.exc_info()[1], |
|
1183 | sys.exc_info()[1], | |
| 1181 | sys.exc_info()[2]) |
|
1184 | sys.exc_info()[2]) | |
| 1182 |
|
1185 | |||
| 1183 | print "***** Error occurred in %s *****" %(procUnitConfObj.name) |
|
1186 | print "***** Error occurred in %s *****" %(procUnitConfObj.name) | |
| 1184 | print "***** %s" %err[-1] |
|
1187 | print "***** %s" %err[-1] | |
| 1185 |
|
1188 | |||
| 1186 | message = "".join(err) |
|
1189 | message = "".join(err) | |
| 1187 |
|
1190 | |||
| 1188 | sys.stderr.write(message) |
|
1191 | sys.stderr.write(message) | |
| 1189 |
|
1192 | |||
| 1190 | if not send_email: |
|
1193 | if not send_email: | |
| 1191 | return |
|
1194 | return | |
| 1192 |
|
1195 | |||
| 1193 | subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name) |
|
1196 | subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name) | |
| 1194 |
|
1197 | |||
| 1195 | subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name) |
|
1198 | subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name) | |
| 1196 | subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname()) |
|
1199 | subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname()) | |
| 1197 | subtitle += "Working directory: %s\n" %os.path.abspath("./") |
|
1200 | subtitle += "Working directory: %s\n" %os.path.abspath("./") | |
| 1198 | subtitle += "Configuration file: %s\n" %self.filename |
|
1201 | subtitle += "Configuration file: %s\n" %self.filename | |
| 1199 | subtitle += "Time: %s\n" %str(datetime.datetime.now()) |
|
1202 | subtitle += "Time: %s\n" %str(datetime.datetime.now()) | |
| 1200 |
|
1203 | |||
| 1201 | readUnitConfObj = self.getReadUnitObj() |
|
1204 | readUnitConfObj = self.getReadUnitObj() | |
| 1202 | if readUnitConfObj: |
|
1205 | if readUnitConfObj: | |
| 1203 | subtitle += "\nInput parameters:\n" |
|
1206 | subtitle += "\nInput parameters:\n" | |
| 1204 | subtitle += "[Data path = %s]\n" %readUnitConfObj.path |
|
1207 | subtitle += "[Data path = %s]\n" %readUnitConfObj.path | |
| 1205 | subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype |
|
1208 | subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype | |
| 1206 | subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate |
|
1209 | subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate | |
| 1207 | subtitle += "[End date = %s]\n" %readUnitConfObj.endDate |
|
1210 | subtitle += "[End date = %s]\n" %readUnitConfObj.endDate | |
| 1208 | subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime |
|
1211 | subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime | |
| 1209 | subtitle += "[End time = %s]\n" %readUnitConfObj.endTime |
|
1212 | subtitle += "[End time = %s]\n" %readUnitConfObj.endTime | |
| 1210 |
|
1213 | |||
| 1211 | adminObj = schainpy.admin.SchainNotify() |
|
1214 | adminObj = schainpy.admin.SchainNotify() | |
| 1212 | adminObj.sendAlert(message=message, |
|
1215 | adminObj.sendAlert(message=message, | |
| 1213 | subject=subject, |
|
1216 | subject=subject, | |
| 1214 | subtitle=subtitle, |
|
1217 | subtitle=subtitle, | |
| 1215 | filename=self.filename) |
|
1218 | filename=self.filename) | |
| 1216 |
|
1219 | |||
| 1217 | def isPaused(self): |
|
1220 | def isPaused(self): | |
| 1218 | return 0 |
|
1221 | return 0 | |
| 1219 |
|
1222 | |||
| 1220 | def isStopped(self): |
|
1223 | def isStopped(self): | |
| 1221 | return 0 |
|
1224 | return 0 | |
| 1222 |
|
1225 | |||
| 1223 | def runController(self): |
|
1226 | def runController(self): | |
| 1224 | """ |
|
1227 | """ | |
| 1225 | returns 0 when this process has been stopped, 1 otherwise |
|
1228 | returns 0 when this process has been stopped, 1 otherwise | |
| 1226 | """ |
|
1229 | """ | |
| 1227 |
|
1230 | |||
| 1228 | if self.isPaused(): |
|
1231 | if self.isPaused(): | |
| 1229 | print "Process suspended" |
|
1232 | print "Process suspended" | |
| 1230 |
|
1233 | |||
| 1231 | while True: |
|
1234 | while True: | |
| 1232 | sleep(0.1) |
|
1235 | sleep(0.1) | |
| 1233 |
|
1236 | |||
| 1234 | if not self.isPaused(): |
|
1237 | if not self.isPaused(): | |
| 1235 | break |
|
1238 | break | |
| 1236 |
|
1239 | |||
| 1237 | if self.isStopped(): |
|
1240 | if self.isStopped(): | |
| 1238 | break |
|
1241 | break | |
| 1239 |
|
1242 | |||
| 1240 | print "Process reinitialized" |
|
1243 | print "Process reinitialized" | |
| 1241 |
|
1244 | |||
| 1242 | if self.isStopped(): |
|
1245 | if self.isStopped(): | |
| 1243 | print "Process stopped" |
|
1246 | print "Process stopped" | |
| 1244 | return 0 |
|
1247 | return 0 | |
| 1245 |
|
1248 | |||
| 1246 | return 1 |
|
1249 | return 1 | |
| 1247 |
|
1250 | |||
| 1248 | def setFilename(self, filename): |
|
1251 | def setFilename(self, filename): | |
| 1249 |
|
1252 | |||
| 1250 | self.filename = filename |
|
1253 | self.filename = filename | |
| 1251 |
|
1254 | |||
| 1252 | def setPlotterQueue(self, plotter_queue): |
|
1255 | def setPlotterQueue(self, plotter_queue): | |
| 1253 |
|
1256 | |||
| 1254 | raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class" |
|
1257 | raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class" | |
| 1255 |
|
1258 | |||
| 1256 | def getPlotterQueue(self): |
|
1259 | def getPlotterQueue(self): | |
| 1257 |
|
1260 | |||
| 1258 | raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class" |
|
1261 | raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class" | |
| 1259 |
|
1262 | |||
| 1260 | def useExternalPlotter(self): |
|
1263 | def useExternalPlotter(self): | |
| 1261 |
|
1264 | |||
| 1262 | raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class" |
|
1265 | raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class" | |
| 1263 |
|
1266 | |||
| 1264 | def run(self): |
|
1267 | def run(self): | |
| 1265 |
|
1268 | |||
| 1266 |
|
1269 | |||
| 1267 | print "*"*60 |
|
1270 | print "*"*60 | |
| 1268 | print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__ |
|
1271 | print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__ | |
| 1269 | print "*"*60 |
|
1272 | print "*"*60 | |
| 1270 |
|
1273 | |||
| 1271 |
|
1274 | |||
| 1272 | keyList = self.procUnitConfObjDict.keys() |
|
1275 | keyList = self.procUnitConfObjDict.keys() | |
| 1273 | keyList.sort() |
|
1276 | keyList.sort() | |
| 1274 |
|
1277 | |||
| 1275 | while(True): |
|
1278 | while(True): | |
| 1276 |
|
1279 | |||
| 1277 | is_ok = False |
|
1280 | is_ok = False | |
| 1278 |
|
1281 | |||
| 1279 | for procKey in keyList: |
|
1282 | for procKey in keyList: | |
| 1280 | # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id) |
|
1283 | # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id) | |
| 1281 |
|
1284 | |||
| 1282 | procUnitConfObj = self.procUnitConfObjDict[procKey] |
|
1285 | procUnitConfObj = self.procUnitConfObjDict[procKey] | |
| 1283 |
|
1286 | |||
| 1284 | try: |
|
1287 | try: | |
| 1285 | sts = procUnitConfObj.run() |
|
1288 | sts = procUnitConfObj.run() | |
| 1286 | is_ok = is_ok or sts |
|
1289 | is_ok = is_ok or sts | |
| 1287 | except KeyboardInterrupt: |
|
1290 | except KeyboardInterrupt: | |
| 1288 | is_ok = False |
|
1291 | is_ok = False | |
| 1289 | break |
|
1292 | break | |
| 1290 | except ValueError, e: |
|
1293 | except ValueError, e: | |
| 1291 | sleep(0.5) |
|
1294 | sleep(0.5) | |
| 1292 | self.__handleError(procUnitConfObj, send_email=True) |
|
1295 | self.__handleError(procUnitConfObj, send_email=True) | |
| 1293 | is_ok = False |
|
1296 | is_ok = False | |
| 1294 | break |
|
1297 | break | |
| 1295 | except: |
|
1298 | except: | |
| 1296 | sleep(0.5) |
|
1299 | sleep(0.5) | |
| 1297 | self.__handleError(procUnitConfObj) |
|
1300 | self.__handleError(procUnitConfObj) | |
| 1298 | is_ok = False |
|
1301 | is_ok = False | |
| 1299 | break |
|
1302 | break | |
| 1300 |
|
1303 | |||
| 1301 | #If every process unit finished so end process |
|
1304 | #If every process unit finished so end process | |
| 1302 | if not(is_ok): |
|
1305 | if not(is_ok): | |
| 1303 | # print "Every process unit have finished" |
|
1306 | # print "Every process unit have finished" | |
| 1304 | break |
|
1307 | break | |
| 1305 |
|
1308 | |||
| 1306 | if not self.runController(): |
|
1309 | if not self.runController(): | |
| 1307 | break |
|
1310 | break | |
| 1308 |
|
1311 | |||
| 1309 | #Closing every process |
|
1312 | #Closing every process | |
| 1310 | for procKey in keyList: |
|
1313 | for procKey in keyList: | |
| 1311 | procUnitConfObj = self.procUnitConfObjDict[procKey] |
|
1314 | procUnitConfObj = self.procUnitConfObjDict[procKey] | |
| 1312 | procUnitConfObj.close() |
|
1315 | procUnitConfObj.close() | |
| 1313 |
|
1316 | |||
| 1314 | print "Process finished" |
|
1317 | print "Process finished" | |
| 1315 |
|
1318 | |||
| 1316 | def start(self, filename=None): |
|
1319 | def start(self, filename=None): | |
| 1317 |
|
1320 | |||
| 1318 | self.writeXml(filename) |
|
1321 | self.writeXml(filename) | |
| 1319 | self.createObjects() |
|
1322 | self.createObjects() | |
| 1320 | self.connectObjects() |
|
1323 | self.connectObjects() | |
| 1321 | self.run() |
|
1324 | self.run() | |
| @@ -1,762 +1,765 | |||||
| 1 | ''' |
|
1 | ''' | |
| 2 |
|
2 | |||
| 3 | $Author: murco $ |
|
3 | $Author: murco $ | |
| 4 | $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $ |
|
4 | $Id: JROHeaderIO.py 151 2012-10-31 19:00:51Z murco $ | |
| 5 | ''' |
|
5 | ''' | |
| 6 | import sys |
|
6 | import sys | |
| 7 | import numpy |
|
7 | import numpy | |
| 8 | import copy |
|
8 | import copy | |
| 9 | import datetime |
|
9 | import datetime | |
| 10 |
|
10 | |||
| 11 | SPEED_OF_LIGHT = 299792458 |
|
11 | SPEED_OF_LIGHT = 299792458 | |
| 12 | SPEED_OF_LIGHT = 3e8 |
|
12 | SPEED_OF_LIGHT = 3e8 | |
| 13 |
|
13 | |||
| 14 | BASIC_STRUCTURE = numpy.dtype([ |
|
14 | BASIC_STRUCTURE = numpy.dtype([ | |
| 15 | ('nSize','<u4'), |
|
15 | ('nSize','<u4'), | |
| 16 | ('nVersion','<u2'), |
|
16 | ('nVersion','<u2'), | |
| 17 | ('nDataBlockId','<u4'), |
|
17 | ('nDataBlockId','<u4'), | |
| 18 | ('nUtime','<u4'), |
|
18 | ('nUtime','<u4'), | |
| 19 | ('nMilsec','<u2'), |
|
19 | ('nMilsec','<u2'), | |
| 20 | ('nTimezone','<i2'), |
|
20 | ('nTimezone','<i2'), | |
| 21 | ('nDstflag','<i2'), |
|
21 | ('nDstflag','<i2'), | |
| 22 | ('nErrorCount','<u4') |
|
22 | ('nErrorCount','<u4') | |
| 23 | ]) |
|
23 | ]) | |
| 24 |
|
24 | |||
| 25 | SYSTEM_STRUCTURE = numpy.dtype([ |
|
25 | SYSTEM_STRUCTURE = numpy.dtype([ | |
| 26 | ('nSize','<u4'), |
|
26 | ('nSize','<u4'), | |
| 27 | ('nNumSamples','<u4'), |
|
27 | ('nNumSamples','<u4'), | |
| 28 | ('nNumProfiles','<u4'), |
|
28 | ('nNumProfiles','<u4'), | |
| 29 | ('nNumChannels','<u4'), |
|
29 | ('nNumChannels','<u4'), | |
| 30 | ('nADCResolution','<u4'), |
|
30 | ('nADCResolution','<u4'), | |
| 31 | ('nPCDIOBusWidth','<u4'), |
|
31 | ('nPCDIOBusWidth','<u4'), | |
| 32 | ]) |
|
32 | ]) | |
| 33 |
|
33 | |||
| 34 | RADAR_STRUCTURE = numpy.dtype([ |
|
34 | RADAR_STRUCTURE = numpy.dtype([ | |
| 35 | ('nSize','<u4'), |
|
35 | ('nSize','<u4'), | |
| 36 | ('nExpType','<u4'), |
|
36 | ('nExpType','<u4'), | |
| 37 | ('nNTx','<u4'), |
|
37 | ('nNTx','<u4'), | |
| 38 | ('fIpp','<f4'), |
|
38 | ('fIpp','<f4'), | |
| 39 | ('fTxA','<f4'), |
|
39 | ('fTxA','<f4'), | |
| 40 | ('fTxB','<f4'), |
|
40 | ('fTxB','<f4'), | |
| 41 | ('nNumWindows','<u4'), |
|
41 | ('nNumWindows','<u4'), | |
| 42 | ('nNumTaus','<u4'), |
|
42 | ('nNumTaus','<u4'), | |
| 43 | ('nCodeType','<u4'), |
|
43 | ('nCodeType','<u4'), | |
| 44 | ('nLine6Function','<u4'), |
|
44 | ('nLine6Function','<u4'), | |
| 45 | ('nLine5Function','<u4'), |
|
45 | ('nLine5Function','<u4'), | |
| 46 | ('fClock','<f4'), |
|
46 | ('fClock','<f4'), | |
| 47 | ('nPrePulseBefore','<u4'), |
|
47 | ('nPrePulseBefore','<u4'), | |
| 48 | ('nPrePulseAfter','<u4'), |
|
48 | ('nPrePulseAfter','<u4'), | |
| 49 | ('sRangeIPP','<a20'), |
|
49 | ('sRangeIPP','<a20'), | |
| 50 | ('sRangeTxA','<a20'), |
|
50 | ('sRangeTxA','<a20'), | |
| 51 | ('sRangeTxB','<a20'), |
|
51 | ('sRangeTxB','<a20'), | |
| 52 | ]) |
|
52 | ]) | |
| 53 |
|
53 | |||
| 54 | SAMPLING_STRUCTURE = numpy.dtype([('h0','<f4'),('dh','<f4'),('nsa','<u4')]) |
|
54 | SAMPLING_STRUCTURE = numpy.dtype([('h0','<f4'),('dh','<f4'),('nsa','<u4')]) | |
| 55 |
|
55 | |||
| 56 |
|
56 | |||
| 57 | PROCESSING_STRUCTURE = numpy.dtype([ |
|
57 | PROCESSING_STRUCTURE = numpy.dtype([ | |
| 58 | ('nSize','<u4'), |
|
58 | ('nSize','<u4'), | |
| 59 | ('nDataType','<u4'), |
|
59 | ('nDataType','<u4'), | |
| 60 | ('nSizeOfDataBlock','<u4'), |
|
60 | ('nSizeOfDataBlock','<u4'), | |
| 61 | ('nProfilesperBlock','<u4'), |
|
61 | ('nProfilesperBlock','<u4'), | |
| 62 | ('nDataBlocksperFile','<u4'), |
|
62 | ('nDataBlocksperFile','<u4'), | |
| 63 | ('nNumWindows','<u4'), |
|
63 | ('nNumWindows','<u4'), | |
| 64 | ('nProcessFlags','<u4'), |
|
64 | ('nProcessFlags','<u4'), | |
| 65 | ('nCoherentIntegrations','<u4'), |
|
65 | ('nCoherentIntegrations','<u4'), | |
| 66 | ('nIncoherentIntegrations','<u4'), |
|
66 | ('nIncoherentIntegrations','<u4'), | |
| 67 | ('nTotalSpectra','<u4') |
|
67 | ('nTotalSpectra','<u4') | |
| 68 | ]) |
|
68 | ]) | |
| 69 |
|
69 | |||
| 70 | class Header(object): |
|
70 | class Header(object): | |
| 71 |
|
71 | |||
| 72 | def __init__(self): |
|
72 | def __init__(self): | |
| 73 | raise NotImplementedError |
|
73 | raise NotImplementedError | |
| 74 |
|
74 | |||
| 75 | def copy(self): |
|
75 | def copy(self): | |
| 76 | return copy.deepcopy(self) |
|
76 | return copy.deepcopy(self) | |
| 77 |
|
77 | |||
| 78 | def read(self): |
|
78 | def read(self): | |
| 79 |
|
79 | |||
| 80 | raise NotImplementedError |
|
80 | raise NotImplementedError | |
| 81 |
|
81 | |||
| 82 | def write(self): |
|
82 | def write(self): | |
| 83 |
|
83 | |||
| 84 | raise NotImplementedError |
|
84 | raise NotImplementedError | |
| 85 |
|
85 | |||
| 86 | def printInfo(self): |
|
86 | def printInfo(self): | |
| 87 |
|
87 | |||
| 88 | message = "#"*50 + "\n" |
|
88 | message = "#"*50 + "\n" | |
| 89 | message += self.__class__.__name__.upper() + "\n" |
|
89 | message += self.__class__.__name__.upper() + "\n" | |
| 90 | message += "#"*50 + "\n" |
|
90 | message += "#"*50 + "\n" | |
| 91 |
|
91 | |||
| 92 | keyList = self.__dict__.keys() |
|
92 | keyList = self.__dict__.keys() | |
| 93 | keyList.sort() |
|
93 | keyList.sort() | |
| 94 |
|
94 | |||
| 95 | for key in keyList: |
|
95 | for key in keyList: | |
| 96 | message += "%s = %s" %(key, self.__dict__[key]) + "\n" |
|
96 | message += "%s = %s" %(key, self.__dict__[key]) + "\n" | |
| 97 |
|
97 | |||
| 98 | if "size" not in keyList: |
|
98 | if "size" not in keyList: | |
| 99 | attr = getattr(self, "size") |
|
99 | attr = getattr(self, "size") | |
| 100 |
|
100 | |||
| 101 | if attr: |
|
101 | if attr: | |
| 102 | message += "%s = %s" %("size", attr) + "\n" |
|
102 | message += "%s = %s" %("size", attr) + "\n" | |
| 103 |
|
103 | |||
| 104 | print message |
|
104 | print message | |
| 105 |
|
105 | |||
| 106 | class BasicHeader(Header): |
|
106 | class BasicHeader(Header): | |
| 107 |
|
107 | |||
| 108 | size = None |
|
108 | size = None | |
| 109 | version = None |
|
109 | version = None | |
| 110 | dataBlock = None |
|
110 | dataBlock = None | |
| 111 | utc = None |
|
111 | utc = None | |
| 112 | ltc = None |
|
112 | ltc = None | |
| 113 | miliSecond = None |
|
113 | miliSecond = None | |
| 114 | timeZone = None |
|
114 | timeZone = None | |
| 115 | dstFlag = None |
|
115 | dstFlag = None | |
| 116 | errorCount = None |
|
116 | errorCount = None | |
| 117 | datatime = None |
|
117 | datatime = None | |
| 118 |
|
||||
| 119 | __LOCALTIME = None |
|
118 | __LOCALTIME = None | |
| 120 |
|
119 | |||
| 121 | def __init__(self, useLocalTime=True): |
|
120 | def __init__(self, useLocalTime=True): | |
| 122 |
|
121 | |||
| 123 | self.size = 24 |
|
122 | self.size = 24 | |
| 124 | self.version = 0 |
|
123 | self.version = 0 | |
| 125 | self.dataBlock = 0 |
|
124 | self.dataBlock = 0 | |
| 126 | self.utc = 0 |
|
125 | self.utc = 0 | |
| 127 | self.miliSecond = 0 |
|
126 | self.miliSecond = 0 | |
| 128 | self.timeZone = 0 |
|
127 | self.timeZone = 0 | |
| 129 | self.dstFlag = 0 |
|
128 | self.dstFlag = 0 | |
| 130 | self.errorCount = 0 |
|
129 | self.errorCount = 0 | |
| 131 |
|
130 | |||
| 132 | self.useLocalTime = useLocalTime |
|
131 | self.useLocalTime = useLocalTime | |
| 133 |
|
132 | |||
| 134 | def read(self, fp): |
|
133 | def read(self, fp): | |
| 135 |
|
134 | |||
| 136 | try: |
|
135 | try: | |
| 137 | header = numpy.fromfile(fp, BASIC_STRUCTURE,1) |
|
136 | if hasattr(fp, 'read'): | |
| 138 |
|
137 | print 'fromfile' | ||
|
|
138 | header = numpy.fromfile(fp, BASIC_STRUCTURE,1) | |||
|
|
139 | else: | |||
|
|
140 | print 'fromstring' | |||
|
|
141 | header = numpy.fromstring(fp, BASIC_STRUCTURE,1) | |||
| 139 | except Exception, e: |
|
142 | except Exception, e: | |
| 140 | print "BasicHeader: " |
|
143 | print "BasicHeader: " | |
| 141 | print e |
|
144 | print e | |
| 142 | return 0 |
|
145 | return 0 | |
| 143 |
|
146 | |||
| 144 | self.size = int(header['nSize'][0]) |
|
147 | self.size = int(header['nSize'][0]) | |
| 145 | self.version = int(header['nVersion'][0]) |
|
148 | self.version = int(header['nVersion'][0]) | |
| 146 | self.dataBlock = int(header['nDataBlockId'][0]) |
|
149 | self.dataBlock = int(header['nDataBlockId'][0]) | |
| 147 | self.utc = int(header['nUtime'][0]) |
|
150 | self.utc = int(header['nUtime'][0]) | |
| 148 | self.miliSecond = int(header['nMilsec'][0]) |
|
151 | self.miliSecond = int(header['nMilsec'][0]) | |
| 149 | self.timeZone = int(header['nTimezone'][0]) |
|
152 | self.timeZone = int(header['nTimezone'][0]) | |
| 150 | self.dstFlag = int(header['nDstflag'][0]) |
|
153 | self.dstFlag = int(header['nDstflag'][0]) | |
| 151 | self.errorCount = int(header['nErrorCount'][0]) |
|
154 | self.errorCount = int(header['nErrorCount'][0]) | |
| 152 |
|
155 | |||
| 153 | if self.size < 24: |
|
156 | if self.size < 24: | |
| 154 | return 0 |
|
157 | return 0 | |
| 155 |
|
158 | |||
| 156 | return 1 |
|
159 | return 1 | |
| 157 |
|
160 | |||
| 158 | def write(self, fp): |
|
161 | def write(self, fp): | |
| 159 |
|
162 | |||
| 160 | headerTuple = (self.size,self.version,self.dataBlock,self.utc,self.miliSecond,self.timeZone,self.dstFlag,self.errorCount) |
|
163 | headerTuple = (self.size,self.version,self.dataBlock,self.utc,self.miliSecond,self.timeZone,self.dstFlag,self.errorCount) | |
| 161 | header = numpy.array(headerTuple, BASIC_STRUCTURE) |
|
164 | header = numpy.array(headerTuple, BASIC_STRUCTURE) | |
| 162 | header.tofile(fp) |
|
165 | header.tofile(fp) | |
| 163 |
|
166 | |||
| 164 | return 1 |
|
167 | return 1 | |
| 165 |
|
168 | |||
| 166 | def get_ltc(self): |
|
169 | def get_ltc(self): | |
| 167 |
|
170 | |||
| 168 | return self.utc - self.timeZone*60 |
|
171 | return self.utc - self.timeZone*60 | |
| 169 |
|
172 | |||
| 170 | def set_ltc(self, value): |
|
173 | def set_ltc(self, value): | |
| 171 |
|
174 | |||
| 172 | self.utc = value + self.timeZone*60 |
|
175 | self.utc = value + self.timeZone*60 | |
| 173 |
|
176 | |||
| 174 | def get_datatime(self): |
|
177 | def get_datatime(self): | |
| 175 |
|
178 | |||
| 176 | return datetime.datetime.utcfromtimestamp(self.ltc) |
|
179 | return datetime.datetime.utcfromtimestamp(self.ltc) | |
| 177 |
|
180 | |||
| 178 | ltc = property(get_ltc, set_ltc) |
|
181 | ltc = property(get_ltc, set_ltc) | |
| 179 | datatime = property(get_datatime) |
|
182 | datatime = property(get_datatime) | |
| 180 |
|
183 | |||
| 181 | class SystemHeader(Header): |
|
184 | class SystemHeader(Header): | |
| 182 |
|
185 | |||
| 183 | size = None |
|
186 | size = None | |
| 184 | nSamples = None |
|
187 | nSamples = None | |
| 185 | nProfiles = None |
|
188 | nProfiles = None | |
| 186 | nChannels = None |
|
189 | nChannels = None | |
| 187 | adcResolution = None |
|
190 | adcResolution = None | |
| 188 | pciDioBusWidth = None |
|
191 | pciDioBusWidth = None | |
| 189 |
|
192 | |||
| 190 | def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWith=0): |
|
193 | def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWith=0): | |
| 191 |
|
194 | |||
| 192 | self.size = 24 |
|
195 | self.size = 24 | |
| 193 | self.nSamples = nSamples |
|
196 | self.nSamples = nSamples | |
| 194 | self.nProfiles = nProfiles |
|
197 | self.nProfiles = nProfiles | |
| 195 | self.nChannels = nChannels |
|
198 | self.nChannels = nChannels | |
| 196 | self.adcResolution = adcResolution |
|
199 | self.adcResolution = adcResolution | |
| 197 | self.pciDioBusWidth = pciDioBusWith |
|
200 | self.pciDioBusWidth = pciDioBusWith | |
| 198 |
|
201 | |||
| 199 | def read(self, fp): |
|
202 | def read(self, fp): | |
| 200 |
|
203 | |||
| 201 | startFp = fp.tell() |
|
204 | startFp = fp.tell() | |
| 202 |
|
205 | |||
| 203 | try: |
|
206 | try: | |
| 204 | header = numpy.fromfile(fp,SYSTEM_STRUCTURE,1) |
|
207 | header = numpy.fromfile(fp,SYSTEM_STRUCTURE,1) | |
| 205 | except Exception, e: |
|
208 | except Exception, e: | |
| 206 | print "System Header: " + e |
|
209 | print "System Header: " + e | |
| 207 | return 0 |
|
210 | return 0 | |
| 208 |
|
211 | |||
| 209 | self.size = header['nSize'][0] |
|
212 | self.size = header['nSize'][0] | |
| 210 | self.nSamples = header['nNumSamples'][0] |
|
213 | self.nSamples = header['nNumSamples'][0] | |
| 211 | self.nProfiles = header['nNumProfiles'][0] |
|
214 | self.nProfiles = header['nNumProfiles'][0] | |
| 212 | self.nChannels = header['nNumChannels'][0] |
|
215 | self.nChannels = header['nNumChannels'][0] | |
| 213 | self.adcResolution = header['nADCResolution'][0] |
|
216 | self.adcResolution = header['nADCResolution'][0] | |
| 214 | self.pciDioBusWidth = header['nPCDIOBusWidth'][0] |
|
217 | self.pciDioBusWidth = header['nPCDIOBusWidth'][0] | |
| 215 |
|
218 | |||
| 216 | endFp = self.size + startFp |
|
219 | endFp = self.size + startFp | |
| 217 |
|
220 | |||
| 218 | if fp.tell() > endFp: |
|
221 | if fp.tell() > endFp: | |
| 219 | sys.stderr.write("Warning %s: Size value read from System Header is lower than it has to be\n" %fp.name) |
|
222 | sys.stderr.write("Warning %s: Size value read from System Header is lower than it has to be\n" %fp.name) | |
| 220 | return 0 |
|
223 | return 0 | |
| 221 |
|
224 | |||
| 222 | if fp.tell() < endFp: |
|
225 | if fp.tell() < endFp: | |
| 223 | sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp.name) |
|
226 | sys.stderr.write("Warning %s: Size value read from System Header size is greater than it has to be\n" %fp.name) | |
| 224 | return 0 |
|
227 | return 0 | |
| 225 |
|
228 | |||
| 226 | return 1 |
|
229 | return 1 | |
| 227 |
|
230 | |||
| 228 | def write(self, fp): |
|
231 | def write(self, fp): | |
| 229 |
|
232 | |||
| 230 | headerTuple = (self.size,self.nSamples,self.nProfiles,self.nChannels,self.adcResolution,self.pciDioBusWidth) |
|
233 | headerTuple = (self.size,self.nSamples,self.nProfiles,self.nChannels,self.adcResolution,self.pciDioBusWidth) | |
| 231 | header = numpy.array(headerTuple,SYSTEM_STRUCTURE) |
|
234 | header = numpy.array(headerTuple,SYSTEM_STRUCTURE) | |
| 232 | header.tofile(fp) |
|
235 | header.tofile(fp) | |
| 233 |
|
236 | |||
| 234 | return 1 |
|
237 | return 1 | |
| 235 |
|
238 | |||
| 236 | class RadarControllerHeader(Header): |
|
239 | class RadarControllerHeader(Header): | |
| 237 |
|
240 | |||
| 238 | expType = None |
|
241 | expType = None | |
| 239 | nTx = None |
|
242 | nTx = None | |
| 240 | ipp = None |
|
243 | ipp = None | |
| 241 | txA = None |
|
244 | txA = None | |
| 242 | txB = None |
|
245 | txB = None | |
| 243 | nWindows = None |
|
246 | nWindows = None | |
| 244 | numTaus = None |
|
247 | numTaus = None | |
| 245 | codeType = None |
|
248 | codeType = None | |
| 246 | line6Function = None |
|
249 | line6Function = None | |
| 247 | line5Function = None |
|
250 | line5Function = None | |
| 248 | fClock = None |
|
251 | fClock = None | |
| 249 | prePulseBefore = None |
|
252 | prePulseBefore = None | |
| 250 | prePulserAfter = None |
|
253 | prePulserAfter = None | |
| 251 | rangeIpp = None |
|
254 | rangeIpp = None | |
| 252 | rangeTxA = None |
|
255 | rangeTxA = None | |
| 253 | rangeTxB = None |
|
256 | rangeTxB = None | |
| 254 |
|
257 | |||
| 255 | __size = None |
|
258 | __size = None | |
| 256 |
|
259 | |||
| 257 | def __init__(self, expType=2, nTx=1, |
|
260 | def __init__(self, expType=2, nTx=1, | |
| 258 | ippKm=None, txA=0, txB=0, |
|
261 | ippKm=None, txA=0, txB=0, | |
| 259 | nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None, |
|
262 | nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None, | |
| 260 | numTaus=0, line6Function=0, line5Function=0, fClock=None, |
|
263 | numTaus=0, line6Function=0, line5Function=0, fClock=None, | |
| 261 | prePulseBefore=0, prePulseAfter=0, |
|
264 | prePulseBefore=0, prePulseAfter=0, | |
| 262 | codeType=0, nCode=0, nBaud=0, code=None, |
|
265 | codeType=0, nCode=0, nBaud=0, code=None, | |
| 263 | flip1=0, flip2=0): |
|
266 | flip1=0, flip2=0): | |
| 264 |
|
267 | |||
| 265 | # self.size = 116 |
|
268 | # self.size = 116 | |
| 266 | self.expType = expType |
|
269 | self.expType = expType | |
| 267 | self.nTx = nTx |
|
270 | self.nTx = nTx | |
| 268 | self.ipp = ippKm |
|
271 | self.ipp = ippKm | |
| 269 | self.txA = txA |
|
272 | self.txA = txA | |
| 270 | self.txB = txB |
|
273 | self.txB = txB | |
| 271 | self.rangeIpp = ippKm |
|
274 | self.rangeIpp = ippKm | |
| 272 | self.rangeTxA = txA |
|
275 | self.rangeTxA = txA | |
| 273 | self.rangeTxB = txB |
|
276 | self.rangeTxB = txB | |
| 274 |
|
277 | |||
| 275 | self.nWindows = nWindows |
|
278 | self.nWindows = nWindows | |
| 276 | self.numTaus = numTaus |
|
279 | self.numTaus = numTaus | |
| 277 | self.codeType = codeType |
|
280 | self.codeType = codeType | |
| 278 | self.line6Function = line6Function |
|
281 | self.line6Function = line6Function | |
| 279 | self.line5Function = line5Function |
|
282 | self.line5Function = line5Function | |
| 280 | self.fClock = fClock |
|
283 | self.fClock = fClock | |
| 281 | self.prePulseBefore = prePulseBefore |
|
284 | self.prePulseBefore = prePulseBefore | |
| 282 | self.prePulserAfter = prePulseAfter |
|
285 | self.prePulserAfter = prePulseAfter | |
| 283 |
|
286 | |||
| 284 | self.nHeights = nHeights |
|
287 | self.nHeights = nHeights | |
| 285 | self.firstHeight = firstHeight |
|
288 | self.firstHeight = firstHeight | |
| 286 | self.deltaHeight = deltaHeight |
|
289 | self.deltaHeight = deltaHeight | |
| 287 | self.samplesWin = nHeights |
|
290 | self.samplesWin = nHeights | |
| 288 |
|
291 | |||
| 289 | self.nCode = nCode |
|
292 | self.nCode = nCode | |
| 290 | self.nBaud = nBaud |
|
293 | self.nBaud = nBaud | |
| 291 | self.code = code |
|
294 | self.code = code | |
| 292 | self.flip1 = flip1 |
|
295 | self.flip1 = flip1 | |
| 293 | self.flip2 = flip2 |
|
296 | self.flip2 = flip2 | |
| 294 |
|
297 | |||
| 295 | self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4 |
|
298 | self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4 | |
| 296 | # self.dynamic = numpy.array([],numpy.dtype('byte')) |
|
299 | # self.dynamic = numpy.array([],numpy.dtype('byte')) | |
| 297 |
|
300 | |||
| 298 | if self.fClock is None and self.deltaHeight is not None: |
|
301 | if self.fClock is None and self.deltaHeight is not None: | |
| 299 | self.fClock = 0.15/(deltaHeight*1e-6) #0.15Km / (height * 1u) |
|
302 | self.fClock = 0.15/(deltaHeight*1e-6) #0.15Km / (height * 1u) | |
| 300 |
|
303 | |||
| 301 | def read(self, fp): |
|
304 | def read(self, fp): | |
| 302 |
|
305 | |||
| 303 |
|
306 | |||
| 304 | startFp = fp.tell() |
|
307 | startFp = fp.tell() | |
| 305 | try: |
|
308 | try: | |
| 306 | header = numpy.fromfile(fp,RADAR_STRUCTURE,1) |
|
309 | header = numpy.fromfile(fp,RADAR_STRUCTURE,1) | |
| 307 | except Exception, e: |
|
310 | except Exception, e: | |
| 308 | print "RadarControllerHeader: " + e |
|
311 | print "RadarControllerHeader: " + e | |
| 309 | return 0 |
|
312 | return 0 | |
| 310 |
|
313 | |||
| 311 | size = int(header['nSize'][0]) |
|
314 | size = int(header['nSize'][0]) | |
| 312 | self.expType = int(header['nExpType'][0]) |
|
315 | self.expType = int(header['nExpType'][0]) | |
| 313 | self.nTx = int(header['nNTx'][0]) |
|
316 | self.nTx = int(header['nNTx'][0]) | |
| 314 | self.ipp = float(header['fIpp'][0]) |
|
317 | self.ipp = float(header['fIpp'][0]) | |
| 315 | self.txA = float(header['fTxA'][0]) |
|
318 | self.txA = float(header['fTxA'][0]) | |
| 316 | self.txB = float(header['fTxB'][0]) |
|
319 | self.txB = float(header['fTxB'][0]) | |
| 317 | self.nWindows = int(header['nNumWindows'][0]) |
|
320 | self.nWindows = int(header['nNumWindows'][0]) | |
| 318 | self.numTaus = int(header['nNumTaus'][0]) |
|
321 | self.numTaus = int(header['nNumTaus'][0]) | |
| 319 | self.codeType = int(header['nCodeType'][0]) |
|
322 | self.codeType = int(header['nCodeType'][0]) | |
| 320 | self.line6Function = int(header['nLine6Function'][0]) |
|
323 | self.line6Function = int(header['nLine6Function'][0]) | |
| 321 | self.line5Function = int(header['nLine5Function'][0]) |
|
324 | self.line5Function = int(header['nLine5Function'][0]) | |
| 322 | self.fClock = float(header['fClock'][0]) |
|
325 | self.fClock = float(header['fClock'][0]) | |
| 323 | self.prePulseBefore = int(header['nPrePulseBefore'][0]) |
|
326 | self.prePulseBefore = int(header['nPrePulseBefore'][0]) | |
| 324 | self.prePulserAfter = int(header['nPrePulseAfter'][0]) |
|
327 | self.prePulserAfter = int(header['nPrePulseAfter'][0]) | |
| 325 | self.rangeIpp = header['sRangeIPP'][0] |
|
328 | self.rangeIpp = header['sRangeIPP'][0] | |
| 326 | self.rangeTxA = header['sRangeTxA'][0] |
|
329 | self.rangeTxA = header['sRangeTxA'][0] | |
| 327 | self.rangeTxB = header['sRangeTxB'][0] |
|
330 | self.rangeTxB = header['sRangeTxB'][0] | |
| 328 |
|
331 | |||
| 329 | samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows) |
|
332 | samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows) | |
| 330 |
|
333 | |||
| 331 | self.nHeights = int(numpy.sum(samplingWindow['nsa'])) |
|
334 | self.nHeights = int(numpy.sum(samplingWindow['nsa'])) | |
| 332 | self.firstHeight = samplingWindow['h0'] |
|
335 | self.firstHeight = samplingWindow['h0'] | |
| 333 | self.deltaHeight = samplingWindow['dh'] |
|
336 | self.deltaHeight = samplingWindow['dh'] | |
| 334 | self.samplesWin = samplingWindow['nsa'] |
|
337 | self.samplesWin = samplingWindow['nsa'] | |
| 335 |
|
338 | |||
| 336 | self.Taus = numpy.fromfile(fp,'<f4',self.numTaus) |
|
339 | self.Taus = numpy.fromfile(fp,'<f4',self.numTaus) | |
| 337 |
|
340 | |||
| 338 | self.code_size = 0 |
|
341 | self.code_size = 0 | |
| 339 | if self.codeType != 0: |
|
342 | if self.codeType != 0: | |
| 340 | self.nCode = int(numpy.fromfile(fp,'<u4',1)) |
|
343 | self.nCode = int(numpy.fromfile(fp,'<u4',1)) | |
| 341 | self.nBaud = int(numpy.fromfile(fp,'<u4',1)) |
|
344 | self.nBaud = int(numpy.fromfile(fp,'<u4',1)) | |
| 342 |
|
345 | |||
| 343 | code = numpy.empty([self.nCode,self.nBaud],dtype='i1') |
|
346 | code = numpy.empty([self.nCode,self.nBaud],dtype='i1') | |
| 344 | for ic in range(self.nCode): |
|
347 | for ic in range(self.nCode): | |
| 345 | temp = numpy.fromfile(fp,'u4',int(numpy.ceil(self.nBaud/32.))) |
|
348 | temp = numpy.fromfile(fp,'u4',int(numpy.ceil(self.nBaud/32.))) | |
| 346 | for ib in range(self.nBaud-1,-1,-1): |
|
349 | for ib in range(self.nBaud-1,-1,-1): | |
| 347 | code[ic,ib] = temp[ib/32]%2 |
|
350 | code[ic,ib] = temp[ib/32]%2 | |
| 348 | temp[ib/32] = temp[ib/32]/2 |
|
351 | temp[ib/32] = temp[ib/32]/2 | |
| 349 |
|
352 | |||
| 350 | self.code = 2.0*code - 1.0 |
|
353 | self.code = 2.0*code - 1.0 | |
| 351 | self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4 |
|
354 | self.code_size = int(numpy.ceil(self.nBaud/32.))*self.nCode*4 | |
| 352 |
|
355 | |||
| 353 | # if self.line5Function == RCfunction.FLIP: |
|
356 | # if self.line5Function == RCfunction.FLIP: | |
| 354 | # self.flip1 = numpy.fromfile(fp,'<u4',1) |
|
357 | # self.flip1 = numpy.fromfile(fp,'<u4',1) | |
| 355 | # |
|
358 | # | |
| 356 | # if self.line6Function == RCfunction.FLIP: |
|
359 | # if self.line6Function == RCfunction.FLIP: | |
| 357 | # self.flip2 = numpy.fromfile(fp,'<u4',1) |
|
360 | # self.flip2 = numpy.fromfile(fp,'<u4',1) | |
| 358 |
|
361 | |||
| 359 | endFp = size + startFp |
|
362 | endFp = size + startFp | |
| 360 |
|
363 | |||
| 361 | if fp.tell() != endFp: |
|
364 | if fp.tell() != endFp: | |
| 362 | # fp.seek(endFp) |
|
365 | # fp.seek(endFp) | |
| 363 | print "%s: Radar Controller Header size is not consistent: from data [%d] != from header field [%d]" %(fp.name, fp.tell()-startFp, size) |
|
366 | print "%s: Radar Controller Header size is not consistent: from data [%d] != from header field [%d]" %(fp.name, fp.tell()-startFp, size) | |
| 364 | # return 0 |
|
367 | # return 0 | |
| 365 |
|
368 | |||
| 366 | if fp.tell() > endFp: |
|
369 | if fp.tell() > endFp: | |
| 367 | sys.stderr.write("Warning %s: Size value read from Radar Controller header is lower than it has to be\n" %fp.name) |
|
370 | sys.stderr.write("Warning %s: Size value read from Radar Controller header is lower than it has to be\n" %fp.name) | |
| 368 | # return 0 |
|
371 | # return 0 | |
| 369 |
|
372 | |||
| 370 | if fp.tell() < endFp: |
|
373 | if fp.tell() < endFp: | |
| 371 | sys.stderr.write("Warning %s: Size value read from Radar Controller header is greater than it has to be\n" %fp.name) |
|
374 | sys.stderr.write("Warning %s: Size value read from Radar Controller header is greater than it has to be\n" %fp.name) | |
| 372 |
|
375 | |||
| 373 |
|
376 | |||
| 374 | return 1 |
|
377 | return 1 | |
| 375 |
|
378 | |||
| 376 | def write(self, fp): |
|
379 | def write(self, fp): | |
| 377 |
|
380 | |||
| 378 | headerTuple = (self.size, |
|
381 | headerTuple = (self.size, | |
| 379 | self.expType, |
|
382 | self.expType, | |
| 380 | self.nTx, |
|
383 | self.nTx, | |
| 381 | self.ipp, |
|
384 | self.ipp, | |
| 382 | self.txA, |
|
385 | self.txA, | |
| 383 | self.txB, |
|
386 | self.txB, | |
| 384 | self.nWindows, |
|
387 | self.nWindows, | |
| 385 | self.numTaus, |
|
388 | self.numTaus, | |
| 386 | self.codeType, |
|
389 | self.codeType, | |
| 387 | self.line6Function, |
|
390 | self.line6Function, | |
| 388 | self.line5Function, |
|
391 | self.line5Function, | |
| 389 | self.fClock, |
|
392 | self.fClock, | |
| 390 | self.prePulseBefore, |
|
393 | self.prePulseBefore, | |
| 391 | self.prePulserAfter, |
|
394 | self.prePulserAfter, | |
| 392 | self.rangeIpp, |
|
395 | self.rangeIpp, | |
| 393 | self.rangeTxA, |
|
396 | self.rangeTxA, | |
| 394 | self.rangeTxB) |
|
397 | self.rangeTxB) | |
| 395 |
|
398 | |||
| 396 | header = numpy.array(headerTuple,RADAR_STRUCTURE) |
|
399 | header = numpy.array(headerTuple,RADAR_STRUCTURE) | |
| 397 | header.tofile(fp) |
|
400 | header.tofile(fp) | |
| 398 |
|
401 | |||
| 399 | sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin) |
|
402 | sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin) | |
| 400 | samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE) |
|
403 | samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE) | |
| 401 | samplingWindow.tofile(fp) |
|
404 | samplingWindow.tofile(fp) | |
| 402 |
|
405 | |||
| 403 | if self.numTaus > 0: |
|
406 | if self.numTaus > 0: | |
| 404 | self.Taus.tofile(fp) |
|
407 | self.Taus.tofile(fp) | |
| 405 |
|
408 | |||
| 406 | if self.codeType !=0: |
|
409 | if self.codeType !=0: | |
| 407 | nCode = numpy.array(self.nCode, '<u4') |
|
410 | nCode = numpy.array(self.nCode, '<u4') | |
| 408 | nCode.tofile(fp) |
|
411 | nCode.tofile(fp) | |
| 409 | nBaud = numpy.array(self.nBaud, '<u4') |
|
412 | nBaud = numpy.array(self.nBaud, '<u4') | |
| 410 | nBaud.tofile(fp) |
|
413 | nBaud.tofile(fp) | |
| 411 | code1 = (self.code + 1.0)/2. |
|
414 | code1 = (self.code + 1.0)/2. | |
| 412 |
|
415 | |||
| 413 | for ic in range(self.nCode): |
|
416 | for ic in range(self.nCode): | |
| 414 | tempx = numpy.zeros(numpy.ceil(self.nBaud/32.)) |
|
417 | tempx = numpy.zeros(numpy.ceil(self.nBaud/32.)) | |
| 415 | start = 0 |
|
418 | start = 0 | |
| 416 | end = 32 |
|
419 | end = 32 | |
| 417 | for i in range(len(tempx)): |
|
420 | for i in range(len(tempx)): | |
| 418 | code_selected = code1[ic,start:end] |
|
421 | code_selected = code1[ic,start:end] | |
| 419 | for j in range(len(code_selected)-1,-1,-1): |
|
422 | for j in range(len(code_selected)-1,-1,-1): | |
| 420 | if code_selected[j] == 1: |
|
423 | if code_selected[j] == 1: | |
| 421 | tempx[i] = tempx[i] + 2**(len(code_selected)-1-j) |
|
424 | tempx[i] = tempx[i] + 2**(len(code_selected)-1-j) | |
| 422 | start = start + 32 |
|
425 | start = start + 32 | |
| 423 | end = end + 32 |
|
426 | end = end + 32 | |
| 424 |
|
427 | |||
| 425 | tempx = tempx.astype('u4') |
|
428 | tempx = tempx.astype('u4') | |
| 426 | tempx.tofile(fp) |
|
429 | tempx.tofile(fp) | |
| 427 |
|
430 | |||
| 428 | # if self.line5Function == RCfunction.FLIP: |
|
431 | # if self.line5Function == RCfunction.FLIP: | |
| 429 | # self.flip1.tofile(fp) |
|
432 | # self.flip1.tofile(fp) | |
| 430 | # |
|
433 | # | |
| 431 | # if self.line6Function == RCfunction.FLIP: |
|
434 | # if self.line6Function == RCfunction.FLIP: | |
| 432 | # self.flip2.tofile(fp) |
|
435 | # self.flip2.tofile(fp) | |
| 433 |
|
436 | |||
| 434 | return 1 |
|
437 | return 1 | |
| 435 |
|
438 | |||
| 436 | def get_ippSeconds(self): |
|
439 | def get_ippSeconds(self): | |
| 437 | ''' |
|
440 | ''' | |
| 438 | ''' |
|
441 | ''' | |
| 439 | ippSeconds = 2.0 * 1000 * self.ipp / SPEED_OF_LIGHT |
|
442 | ippSeconds = 2.0 * 1000 * self.ipp / SPEED_OF_LIGHT | |
| 440 |
|
443 | |||
| 441 | return ippSeconds |
|
444 | return ippSeconds | |
| 442 |
|
445 | |||
| 443 | def set_ippSeconds(self, ippSeconds): |
|
446 | def set_ippSeconds(self, ippSeconds): | |
| 444 | ''' |
|
447 | ''' | |
| 445 | ''' |
|
448 | ''' | |
| 446 |
|
449 | |||
| 447 | self.ipp = ippSeconds * SPEED_OF_LIGHT / (2.0*1000) |
|
450 | self.ipp = ippSeconds * SPEED_OF_LIGHT / (2.0*1000) | |
| 448 |
|
451 | |||
| 449 | return |
|
452 | return | |
| 450 |
|
453 | |||
| 451 | def get_size(self): |
|
454 | def get_size(self): | |
| 452 |
|
455 | |||
| 453 | self.__size = 116 + 12*self.nWindows + 4*self.numTaus |
|
456 | self.__size = 116 + 12*self.nWindows + 4*self.numTaus | |
| 454 |
|
457 | |||
| 455 | if self.codeType != 0: |
|
458 | if self.codeType != 0: | |
| 456 | self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.) |
|
459 | self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.) | |
| 457 |
|
460 | |||
| 458 | return self.__size |
|
461 | return self.__size | |
| 459 |
|
462 | |||
| 460 | def set_size(self, value): |
|
463 | def set_size(self, value): | |
| 461 |
|
464 | |||
| 462 | raise IOError, "size is a property and it cannot be set, just read" |
|
465 | raise IOError, "size is a property and it cannot be set, just read" | |
| 463 |
|
466 | |||
| 464 | return |
|
467 | return | |
| 465 |
|
468 | |||
| 466 | ippSeconds = property(get_ippSeconds, set_ippSeconds) |
|
469 | ippSeconds = property(get_ippSeconds, set_ippSeconds) | |
| 467 | size = property(get_size, set_size) |
|
470 | size = property(get_size, set_size) | |
| 468 |
|
471 | |||
| 469 | class ProcessingHeader(Header): |
|
472 | class ProcessingHeader(Header): | |
| 470 |
|
473 | |||
| 471 | # size = None |
|
474 | # size = None | |
| 472 | dtype = None |
|
475 | dtype = None | |
| 473 | blockSize = None |
|
476 | blockSize = None | |
| 474 | profilesPerBlock = None |
|
477 | profilesPerBlock = None | |
| 475 | dataBlocksPerFile = None |
|
478 | dataBlocksPerFile = None | |
| 476 | nWindows = None |
|
479 | nWindows = None | |
| 477 | processFlags = None |
|
480 | processFlags = None | |
| 478 | nCohInt = None |
|
481 | nCohInt = None | |
| 479 | nIncohInt = None |
|
482 | nIncohInt = None | |
| 480 | totalSpectra = None |
|
483 | totalSpectra = None | |
| 481 |
|
484 | |||
| 482 | flag_dc = None |
|
485 | flag_dc = None | |
| 483 | flag_cspc = None |
|
486 | flag_cspc = None | |
| 484 |
|
487 | |||
| 485 | def __init__(self): |
|
488 | def __init__(self): | |
| 486 |
|
489 | |||
| 487 | # self.size = 0 |
|
490 | # self.size = 0 | |
| 488 | self.dtype = 0 |
|
491 | self.dtype = 0 | |
| 489 | self.blockSize = 0 |
|
492 | self.blockSize = 0 | |
| 490 | self.profilesPerBlock = 0 |
|
493 | self.profilesPerBlock = 0 | |
| 491 | self.dataBlocksPerFile = 0 |
|
494 | self.dataBlocksPerFile = 0 | |
| 492 | self.nWindows = 0 |
|
495 | self.nWindows = 0 | |
| 493 | self.processFlags = 0 |
|
496 | self.processFlags = 0 | |
| 494 | self.nCohInt = 0 |
|
497 | self.nCohInt = 0 | |
| 495 | self.nIncohInt = 0 |
|
498 | self.nIncohInt = 0 | |
| 496 | self.totalSpectra = 0 |
|
499 | self.totalSpectra = 0 | |
| 497 |
|
500 | |||
| 498 | self.nHeights = 0 |
|
501 | self.nHeights = 0 | |
| 499 | self.firstHeight = 0 |
|
502 | self.firstHeight = 0 | |
| 500 | self.deltaHeight = 0 |
|
503 | self.deltaHeight = 0 | |
| 501 | self.samplesWin = 0 |
|
504 | self.samplesWin = 0 | |
| 502 | self.spectraComb = 0 |
|
505 | self.spectraComb = 0 | |
| 503 | self.nCode = None |
|
506 | self.nCode = None | |
| 504 | self.code = None |
|
507 | self.code = None | |
| 505 | self.nBaud = None |
|
508 | self.nBaud = None | |
| 506 |
|
509 | |||
| 507 | self.shif_fft = False |
|
510 | self.shif_fft = False | |
| 508 | self.flag_dc = False |
|
511 | self.flag_dc = False | |
| 509 | self.flag_cspc = False |
|
512 | self.flag_cspc = False | |
| 510 | self.flag_decode = False |
|
513 | self.flag_decode = False | |
| 511 | self.flag_deflip = False |
|
514 | self.flag_deflip = False | |
| 512 |
|
515 | |||
| 513 | def read(self, fp): |
|
516 | def read(self, fp): | |
| 514 |
|
517 | |||
| 515 | startFp = fp.tell() |
|
518 | startFp = fp.tell() | |
| 516 |
|
519 | |||
| 517 | try: |
|
520 | try: | |
| 518 | header = numpy.fromfile(fp,PROCESSING_STRUCTURE,1) |
|
521 | header = numpy.fromfile(fp,PROCESSING_STRUCTURE,1) | |
| 519 | except Exception, e: |
|
522 | except Exception, e: | |
| 520 | print "ProcessingHeader: " + e |
|
523 | print "ProcessingHeader: " + e | |
| 521 | return 0 |
|
524 | return 0 | |
| 522 |
|
525 | |||
| 523 | size = int(header['nSize'][0]) |
|
526 | size = int(header['nSize'][0]) | |
| 524 | self.dtype = int(header['nDataType'][0]) |
|
527 | self.dtype = int(header['nDataType'][0]) | |
| 525 | self.blockSize = int(header['nSizeOfDataBlock'][0]) |
|
528 | self.blockSize = int(header['nSizeOfDataBlock'][0]) | |
| 526 | self.profilesPerBlock = int(header['nProfilesperBlock'][0]) |
|
529 | self.profilesPerBlock = int(header['nProfilesperBlock'][0]) | |
| 527 | self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0]) |
|
530 | self.dataBlocksPerFile = int(header['nDataBlocksperFile'][0]) | |
| 528 | self.nWindows = int(header['nNumWindows'][0]) |
|
531 | self.nWindows = int(header['nNumWindows'][0]) | |
| 529 | self.processFlags = header['nProcessFlags'] |
|
532 | self.processFlags = header['nProcessFlags'] | |
| 530 | self.nCohInt = int(header['nCoherentIntegrations'][0]) |
|
533 | self.nCohInt = int(header['nCoherentIntegrations'][0]) | |
| 531 | self.nIncohInt = int(header['nIncoherentIntegrations'][0]) |
|
534 | self.nIncohInt = int(header['nIncoherentIntegrations'][0]) | |
| 532 | self.totalSpectra = int(header['nTotalSpectra'][0]) |
|
535 | self.totalSpectra = int(header['nTotalSpectra'][0]) | |
| 533 |
|
536 | |||
| 534 | samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows) |
|
537 | samplingWindow = numpy.fromfile(fp,SAMPLING_STRUCTURE,self.nWindows) | |
| 535 |
|
538 | |||
| 536 | self.nHeights = int(numpy.sum(samplingWindow['nsa'])) |
|
539 | self.nHeights = int(numpy.sum(samplingWindow['nsa'])) | |
| 537 | self.firstHeight = float(samplingWindow['h0'][0]) |
|
540 | self.firstHeight = float(samplingWindow['h0'][0]) | |
| 538 | self.deltaHeight = float(samplingWindow['dh'][0]) |
|
541 | self.deltaHeight = float(samplingWindow['dh'][0]) | |
| 539 | self.samplesWin = samplingWindow['nsa'][0] |
|
542 | self.samplesWin = samplingWindow['nsa'][0] | |
| 540 |
|
543 | |||
| 541 | self.spectraComb = numpy.fromfile(fp,'u1',2*self.totalSpectra) |
|
544 | self.spectraComb = numpy.fromfile(fp,'u1',2*self.totalSpectra) | |
| 542 |
|
545 | |||
| 543 | if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE): |
|
546 | if ((self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE) == PROCFLAG.DEFINE_PROCESS_CODE): | |
| 544 | self.nCode = int(numpy.fromfile(fp,'<u4',1)) |
|
547 | self.nCode = int(numpy.fromfile(fp,'<u4',1)) | |
| 545 | self.nBaud = int(numpy.fromfile(fp,'<u4',1)) |
|
548 | self.nBaud = int(numpy.fromfile(fp,'<u4',1)) | |
| 546 | self.code = numpy.fromfile(fp,'<f4',self.nCode*self.nBaud).reshape(self.nCode,self.nBaud) |
|
549 | self.code = numpy.fromfile(fp,'<f4',self.nCode*self.nBaud).reshape(self.nCode,self.nBaud) | |
| 547 |
|
550 | |||
| 548 | if ((self.processFlags & PROCFLAG.EXP_NAME_ESP) == PROCFLAG.EXP_NAME_ESP): |
|
551 | if ((self.processFlags & PROCFLAG.EXP_NAME_ESP) == PROCFLAG.EXP_NAME_ESP): | |
| 549 | exp_name_len = int(numpy.fromfile(fp,'<u4',1)) |
|
552 | exp_name_len = int(numpy.fromfile(fp,'<u4',1)) | |
| 550 | exp_name = numpy.fromfile(fp,'u1',exp_name_len+1) |
|
553 | exp_name = numpy.fromfile(fp,'u1',exp_name_len+1) | |
| 551 |
|
554 | |||
| 552 | if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA): |
|
555 | if ((self.processFlags & PROCFLAG.SHIFT_FFT_DATA) == PROCFLAG.SHIFT_FFT_DATA): | |
| 553 | self.shif_fft = True |
|
556 | self.shif_fft = True | |
| 554 | else: |
|
557 | else: | |
| 555 | self.shif_fft = False |
|
558 | self.shif_fft = False | |
| 556 |
|
559 | |||
| 557 | if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC): |
|
560 | if ((self.processFlags & PROCFLAG.SAVE_CHANNELS_DC) == PROCFLAG.SAVE_CHANNELS_DC): | |
| 558 | self.flag_dc = True |
|
561 | self.flag_dc = True | |
| 559 | else: |
|
562 | else: | |
| 560 | self.flag_dc = False |
|
563 | self.flag_dc = False | |
| 561 |
|
564 | |||
| 562 | if ((self.processFlags & PROCFLAG.DECODE_DATA) == PROCFLAG.DECODE_DATA): |
|
565 | if ((self.processFlags & PROCFLAG.DECODE_DATA) == PROCFLAG.DECODE_DATA): | |
| 563 | self.flag_decode = True |
|
566 | self.flag_decode = True | |
| 564 | else: |
|
567 | else: | |
| 565 | self.flag_decode = False |
|
568 | self.flag_decode = False | |
| 566 |
|
569 | |||
| 567 | if ((self.processFlags & PROCFLAG.DEFLIP_DATA) == PROCFLAG.DEFLIP_DATA): |
|
570 | if ((self.processFlags & PROCFLAG.DEFLIP_DATA) == PROCFLAG.DEFLIP_DATA): | |
| 568 | self.flag_deflip = True |
|
571 | self.flag_deflip = True | |
| 569 | else: |
|
572 | else: | |
| 570 | self.flag_deflip = False |
|
573 | self.flag_deflip = False | |
| 571 |
|
574 | |||
| 572 | nChannels = 0 |
|
575 | nChannels = 0 | |
| 573 | nPairs = 0 |
|
576 | nPairs = 0 | |
| 574 | pairList = [] |
|
577 | pairList = [] | |
| 575 |
|
578 | |||
| 576 | for i in range( 0, self.totalSpectra*2, 2 ): |
|
579 | for i in range( 0, self.totalSpectra*2, 2 ): | |
| 577 | if self.spectraComb[i] == self.spectraComb[i+1]: |
|
580 | if self.spectraComb[i] == self.spectraComb[i+1]: | |
| 578 | nChannels = nChannels + 1 #par de canales iguales |
|
581 | nChannels = nChannels + 1 #par de canales iguales | |
| 579 | else: |
|
582 | else: | |
| 580 | nPairs = nPairs + 1 #par de canales diferentes |
|
583 | nPairs = nPairs + 1 #par de canales diferentes | |
| 581 | pairList.append( (self.spectraComb[i], self.spectraComb[i+1]) ) |
|
584 | pairList.append( (self.spectraComb[i], self.spectraComb[i+1]) ) | |
| 582 |
|
585 | |||
| 583 | self.flag_cspc = False |
|
586 | self.flag_cspc = False | |
| 584 | if nPairs > 0: |
|
587 | if nPairs > 0: | |
| 585 | self.flag_cspc = True |
|
588 | self.flag_cspc = True | |
| 586 |
|
589 | |||
| 587 | endFp = size + startFp |
|
590 | endFp = size + startFp | |
| 588 |
|
591 | |||
| 589 | if fp.tell() > endFp: |
|
592 | if fp.tell() > endFp: | |
| 590 | sys.stderr.write("Warning: Processing header size is lower than it has to be") |
|
593 | sys.stderr.write("Warning: Processing header size is lower than it has to be") | |
| 591 | return 0 |
|
594 | return 0 | |
| 592 |
|
595 | |||
| 593 | if fp.tell() < endFp: |
|
596 | if fp.tell() < endFp: | |
| 594 | sys.stderr.write("Warning: Processing header size is greater than it is considered") |
|
597 | sys.stderr.write("Warning: Processing header size is greater than it is considered") | |
| 595 |
|
598 | |||
| 596 | return 1 |
|
599 | return 1 | |
| 597 |
|
600 | |||
| 598 | def write(self, fp): |
|
601 | def write(self, fp): | |
| 599 | #Clear DEFINE_PROCESS_CODE |
|
602 | #Clear DEFINE_PROCESS_CODE | |
| 600 | self.processFlags = self.processFlags & (~PROCFLAG.DEFINE_PROCESS_CODE) |
|
603 | self.processFlags = self.processFlags & (~PROCFLAG.DEFINE_PROCESS_CODE) | |
| 601 |
|
604 | |||
| 602 | headerTuple = (self.size, |
|
605 | headerTuple = (self.size, | |
| 603 | self.dtype, |
|
606 | self.dtype, | |
| 604 | self.blockSize, |
|
607 | self.blockSize, | |
| 605 | self.profilesPerBlock, |
|
608 | self.profilesPerBlock, | |
| 606 | self.dataBlocksPerFile, |
|
609 | self.dataBlocksPerFile, | |
| 607 | self.nWindows, |
|
610 | self.nWindows, | |
| 608 | self.processFlags, |
|
611 | self.processFlags, | |
| 609 | self.nCohInt, |
|
612 | self.nCohInt, | |
| 610 | self.nIncohInt, |
|
613 | self.nIncohInt, | |
| 611 | self.totalSpectra) |
|
614 | self.totalSpectra) | |
| 612 |
|
615 | |||
| 613 | header = numpy.array(headerTuple,PROCESSING_STRUCTURE) |
|
616 | header = numpy.array(headerTuple,PROCESSING_STRUCTURE) | |
| 614 | header.tofile(fp) |
|
617 | header.tofile(fp) | |
| 615 |
|
618 | |||
| 616 | if self.nWindows != 0: |
|
619 | if self.nWindows != 0: | |
| 617 | sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin) |
|
620 | sampleWindowTuple = (self.firstHeight,self.deltaHeight,self.samplesWin) | |
| 618 | samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE) |
|
621 | samplingWindow = numpy.array(sampleWindowTuple,SAMPLING_STRUCTURE) | |
| 619 | samplingWindow.tofile(fp) |
|
622 | samplingWindow.tofile(fp) | |
| 620 |
|
623 | |||
| 621 | if self.totalSpectra != 0: |
|
624 | if self.totalSpectra != 0: | |
| 622 | # spectraComb = numpy.array([],numpy.dtype('u1')) |
|
625 | # spectraComb = numpy.array([],numpy.dtype('u1')) | |
| 623 | spectraComb = self.spectraComb |
|
626 | spectraComb = self.spectraComb | |
| 624 | spectraComb.tofile(fp) |
|
627 | spectraComb.tofile(fp) | |
| 625 |
|
628 | |||
| 626 | # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE: |
|
629 | # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE: | |
| 627 | # nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba |
|
630 | # nCode = numpy.array([self.nCode], numpy.dtype('u4')) #Probar con un dato que almacene codigo, hasta el momento no se hizo la prueba | |
| 628 | # nCode.tofile(fp) |
|
631 | # nCode.tofile(fp) | |
| 629 | # |
|
632 | # | |
| 630 | # nBaud = numpy.array([self.nBaud], numpy.dtype('u4')) |
|
633 | # nBaud = numpy.array([self.nBaud], numpy.dtype('u4')) | |
| 631 | # nBaud.tofile(fp) |
|
634 | # nBaud.tofile(fp) | |
| 632 | # |
|
635 | # | |
| 633 | # code = self.code.reshape(self.nCode*self.nBaud) |
|
636 | # code = self.code.reshape(self.nCode*self.nBaud) | |
| 634 | # code = code.astype(numpy.dtype('<f4')) |
|
637 | # code = code.astype(numpy.dtype('<f4')) | |
| 635 | # code.tofile(fp) |
|
638 | # code.tofile(fp) | |
| 636 |
|
639 | |||
| 637 | return 1 |
|
640 | return 1 | |
| 638 |
|
641 | |||
| 639 | def get_size(self): |
|
642 | def get_size(self): | |
| 640 |
|
643 | |||
| 641 | self.__size = 40 + 12*self.nWindows + 2*self.totalSpectra |
|
644 | self.__size = 40 + 12*self.nWindows + 2*self.totalSpectra | |
| 642 |
|
645 | |||
| 643 | # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE: |
|
646 | # if self.processFlags & PROCFLAG.DEFINE_PROCESS_CODE == PROCFLAG.DEFINE_PROCESS_CODE: | |
| 644 | # self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.) |
|
647 | # self.__size += 4 + 4 + 4*self.nCode*numpy.ceil(self.nBaud/32.) | |
| 645 | # self.__size += 4 + 4 + 4 * self.nCode * self.nBaud |
|
648 | # self.__size += 4 + 4 + 4 * self.nCode * self.nBaud | |
| 646 |
|
649 | |||
| 647 | return self.__size |
|
650 | return self.__size | |
| 648 |
|
651 | |||
| 649 | def set_size(self, value): |
|
652 | def set_size(self, value): | |
| 650 |
|
653 | |||
| 651 | raise IOError, "size is a property and it cannot be set, just read" |
|
654 | raise IOError, "size is a property and it cannot be set, just read" | |
| 652 |
|
655 | |||
| 653 | return |
|
656 | return | |
| 654 |
|
657 | |||
| 655 | size = property(get_size, set_size) |
|
658 | size = property(get_size, set_size) | |
| 656 |
|
659 | |||
| 657 | class RCfunction: |
|
660 | class RCfunction: | |
| 658 | NONE=0 |
|
661 | NONE=0 | |
| 659 | FLIP=1 |
|
662 | FLIP=1 | |
| 660 | CODE=2 |
|
663 | CODE=2 | |
| 661 | SAMPLING=3 |
|
664 | SAMPLING=3 | |
| 662 | LIN6DIV256=4 |
|
665 | LIN6DIV256=4 | |
| 663 | SYNCHRO=5 |
|
666 | SYNCHRO=5 | |
| 664 |
|
667 | |||
| 665 | class nCodeType: |
|
668 | class nCodeType: | |
| 666 | NONE=0 |
|
669 | NONE=0 | |
| 667 | USERDEFINE=1 |
|
670 | USERDEFINE=1 | |
| 668 | BARKER2=2 |
|
671 | BARKER2=2 | |
| 669 | BARKER3=3 |
|
672 | BARKER3=3 | |
| 670 | BARKER4=4 |
|
673 | BARKER4=4 | |
| 671 | BARKER5=5 |
|
674 | BARKER5=5 | |
| 672 | BARKER7=6 |
|
675 | BARKER7=6 | |
| 673 | BARKER11=7 |
|
676 | BARKER11=7 | |
| 674 | BARKER13=8 |
|
677 | BARKER13=8 | |
| 675 | AC128=9 |
|
678 | AC128=9 | |
| 676 | COMPLEMENTARYCODE2=10 |
|
679 | COMPLEMENTARYCODE2=10 | |
| 677 | COMPLEMENTARYCODE4=11 |
|
680 | COMPLEMENTARYCODE4=11 | |
| 678 | COMPLEMENTARYCODE8=12 |
|
681 | COMPLEMENTARYCODE8=12 | |
| 679 | COMPLEMENTARYCODE16=13 |
|
682 | COMPLEMENTARYCODE16=13 | |
| 680 | COMPLEMENTARYCODE32=14 |
|
683 | COMPLEMENTARYCODE32=14 | |
| 681 | COMPLEMENTARYCODE64=15 |
|
684 | COMPLEMENTARYCODE64=15 | |
| 682 | COMPLEMENTARYCODE128=16 |
|
685 | COMPLEMENTARYCODE128=16 | |
| 683 | CODE_BINARY28=17 |
|
686 | CODE_BINARY28=17 | |
| 684 |
|
687 | |||
| 685 | class PROCFLAG: |
|
688 | class PROCFLAG: | |
| 686 |
|
689 | |||
| 687 | COHERENT_INTEGRATION = numpy.uint32(0x00000001) |
|
690 | COHERENT_INTEGRATION = numpy.uint32(0x00000001) | |
| 688 | DECODE_DATA = numpy.uint32(0x00000002) |
|
691 | DECODE_DATA = numpy.uint32(0x00000002) | |
| 689 | SPECTRA_CALC = numpy.uint32(0x00000004) |
|
692 | SPECTRA_CALC = numpy.uint32(0x00000004) | |
| 690 | INCOHERENT_INTEGRATION = numpy.uint32(0x00000008) |
|
693 | INCOHERENT_INTEGRATION = numpy.uint32(0x00000008) | |
| 691 | POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010) |
|
694 | POST_COHERENT_INTEGRATION = numpy.uint32(0x00000010) | |
| 692 | SHIFT_FFT_DATA = numpy.uint32(0x00000020) |
|
695 | SHIFT_FFT_DATA = numpy.uint32(0x00000020) | |
| 693 |
|
696 | |||
| 694 | DATATYPE_CHAR = numpy.uint32(0x00000040) |
|
697 | DATATYPE_CHAR = numpy.uint32(0x00000040) | |
| 695 | DATATYPE_SHORT = numpy.uint32(0x00000080) |
|
698 | DATATYPE_SHORT = numpy.uint32(0x00000080) | |
| 696 | DATATYPE_LONG = numpy.uint32(0x00000100) |
|
699 | DATATYPE_LONG = numpy.uint32(0x00000100) | |
| 697 | DATATYPE_INT64 = numpy.uint32(0x00000200) |
|
700 | DATATYPE_INT64 = numpy.uint32(0x00000200) | |
| 698 | DATATYPE_FLOAT = numpy.uint32(0x00000400) |
|
701 | DATATYPE_FLOAT = numpy.uint32(0x00000400) | |
| 699 | DATATYPE_DOUBLE = numpy.uint32(0x00000800) |
|
702 | DATATYPE_DOUBLE = numpy.uint32(0x00000800) | |
| 700 |
|
703 | |||
| 701 | DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000) |
|
704 | DATAARRANGE_CONTIGUOUS_CH = numpy.uint32(0x00001000) | |
| 702 | DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000) |
|
705 | DATAARRANGE_CONTIGUOUS_H = numpy.uint32(0x00002000) | |
| 703 | DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000) |
|
706 | DATAARRANGE_CONTIGUOUS_P = numpy.uint32(0x00004000) | |
| 704 |
|
707 | |||
| 705 | SAVE_CHANNELS_DC = numpy.uint32(0x00008000) |
|
708 | SAVE_CHANNELS_DC = numpy.uint32(0x00008000) | |
| 706 | DEFLIP_DATA = numpy.uint32(0x00010000) |
|
709 | DEFLIP_DATA = numpy.uint32(0x00010000) | |
| 707 | DEFINE_PROCESS_CODE = numpy.uint32(0x00020000) |
|
710 | DEFINE_PROCESS_CODE = numpy.uint32(0x00020000) | |
| 708 |
|
711 | |||
| 709 | ACQ_SYS_NATALIA = numpy.uint32(0x00040000) |
|
712 | ACQ_SYS_NATALIA = numpy.uint32(0x00040000) | |
| 710 | ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000) |
|
713 | ACQ_SYS_ECHOTEK = numpy.uint32(0x00080000) | |
| 711 | ACQ_SYS_ADRXD = numpy.uint32(0x000C0000) |
|
714 | ACQ_SYS_ADRXD = numpy.uint32(0x000C0000) | |
| 712 | ACQ_SYS_JULIA = numpy.uint32(0x00100000) |
|
715 | ACQ_SYS_JULIA = numpy.uint32(0x00100000) | |
| 713 | ACQ_SYS_XXXXXX = numpy.uint32(0x00140000) |
|
716 | ACQ_SYS_XXXXXX = numpy.uint32(0x00140000) | |
| 714 |
|
717 | |||
| 715 | EXP_NAME_ESP = numpy.uint32(0x00200000) |
|
718 | EXP_NAME_ESP = numpy.uint32(0x00200000) | |
| 716 | CHANNEL_NAMES_ESP = numpy.uint32(0x00400000) |
|
719 | CHANNEL_NAMES_ESP = numpy.uint32(0x00400000) | |
| 717 |
|
720 | |||
| 718 | OPERATION_MASK = numpy.uint32(0x0000003F) |
|
721 | OPERATION_MASK = numpy.uint32(0x0000003F) | |
| 719 | DATATYPE_MASK = numpy.uint32(0x00000FC0) |
|
722 | DATATYPE_MASK = numpy.uint32(0x00000FC0) | |
| 720 | DATAARRANGE_MASK = numpy.uint32(0x00007000) |
|
723 | DATAARRANGE_MASK = numpy.uint32(0x00007000) | |
| 721 | ACQ_SYS_MASK = numpy.uint32(0x001C0000) |
|
724 | ACQ_SYS_MASK = numpy.uint32(0x001C0000) | |
| 722 |
|
725 | |||
| 723 | dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')]) |
|
726 | dtype0 = numpy.dtype([('real','<i1'),('imag','<i1')]) | |
| 724 | dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')]) |
|
727 | dtype1 = numpy.dtype([('real','<i2'),('imag','<i2')]) | |
| 725 | dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')]) |
|
728 | dtype2 = numpy.dtype([('real','<i4'),('imag','<i4')]) | |
| 726 | dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')]) |
|
729 | dtype3 = numpy.dtype([('real','<i8'),('imag','<i8')]) | |
| 727 | dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')]) |
|
730 | dtype4 = numpy.dtype([('real','<f4'),('imag','<f4')]) | |
| 728 | dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')]) |
|
731 | dtype5 = numpy.dtype([('real','<f8'),('imag','<f8')]) | |
| 729 |
|
732 | |||
| 730 | NUMPY_DTYPE_LIST = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5] |
|
733 | NUMPY_DTYPE_LIST = [dtype0, dtype1, dtype2, dtype3, dtype4, dtype5] | |
| 731 |
|
734 | |||
| 732 | PROCFLAG_DTYPE_LIST = [PROCFLAG.DATATYPE_CHAR, |
|
735 | PROCFLAG_DTYPE_LIST = [PROCFLAG.DATATYPE_CHAR, | |
| 733 | PROCFLAG.DATATYPE_SHORT, |
|
736 | PROCFLAG.DATATYPE_SHORT, | |
| 734 | PROCFLAG.DATATYPE_LONG, |
|
737 | PROCFLAG.DATATYPE_LONG, | |
| 735 | PROCFLAG.DATATYPE_INT64, |
|
738 | PROCFLAG.DATATYPE_INT64, | |
| 736 | PROCFLAG.DATATYPE_FLOAT, |
|
739 | PROCFLAG.DATATYPE_FLOAT, | |
| 737 | PROCFLAG.DATATYPE_DOUBLE] |
|
740 | PROCFLAG.DATATYPE_DOUBLE] | |
| 738 |
|
741 | |||
| 739 | DTYPE_WIDTH = [1, 2, 4, 8, 4, 8] |
|
742 | DTYPE_WIDTH = [1, 2, 4, 8, 4, 8] | |
| 740 |
|
743 | |||
| 741 | def get_dtype_index(numpy_dtype): |
|
744 | def get_dtype_index(numpy_dtype): | |
| 742 |
|
745 | |||
| 743 | index = None |
|
746 | index = None | |
| 744 |
|
747 | |||
| 745 | for i in range(len(NUMPY_DTYPE_LIST)): |
|
748 | for i in range(len(NUMPY_DTYPE_LIST)): | |
| 746 | if numpy_dtype == NUMPY_DTYPE_LIST[i]: |
|
749 | if numpy_dtype == NUMPY_DTYPE_LIST[i]: | |
| 747 | index = i |
|
750 | index = i | |
| 748 | break |
|
751 | break | |
| 749 |
|
752 | |||
| 750 | return index |
|
753 | return index | |
| 751 |
|
754 | |||
| 752 | def get_numpy_dtype(index): |
|
755 | def get_numpy_dtype(index): | |
| 753 |
|
756 | |||
| 754 | return NUMPY_DTYPE_LIST[index] |
|
757 | return NUMPY_DTYPE_LIST[index] | |
| 755 |
|
758 | |||
| 756 | def get_procflag_dtype(index): |
|
759 | def get_procflag_dtype(index): | |
| 757 |
|
760 | |||
| 758 | return PROCFLAG_DTYPE_LIST[index] |
|
761 | return PROCFLAG_DTYPE_LIST[index] | |
| 759 |
|
762 | |||
| 760 | def get_dtype_width(index): |
|
763 | def get_dtype_width(index): | |
| 761 |
|
764 | |||
| 762 | return DTYPE_WIDTH[index] No newline at end of file |
|
765 | return DTYPE_WIDTH[index] | |
| @@ -1,1810 +1,1814 | |||||
| 1 | ''' |
|
1 | ''' | |
| 2 | Created on Jul 2, 2014 |
|
2 | Created on Jul 2, 2014 | |
| 3 |
|
3 | |||
| 4 | @author: roj-idl71 |
|
4 | @author: roj-idl71 | |
| 5 | ''' |
|
5 | ''' | |
| 6 | import os |
|
6 | import os | |
| 7 | import sys |
|
7 | import sys | |
| 8 | import glob |
|
8 | import glob | |
| 9 | import time |
|
9 | import time | |
| 10 | import numpy |
|
10 | import numpy | |
| 11 | import fnmatch |
|
11 | import fnmatch | |
| 12 | import inspect |
|
12 | import inspect | |
| 13 | import time, datetime |
|
13 | import time, datetime | |
| 14 | import traceback |
|
14 | import traceback | |
|
|
15 | import zmq | |||
| 15 |
|
16 | |||
| 16 | try: |
|
17 | try: | |
| 17 | from gevent import sleep |
|
18 | from gevent import sleep | |
| 18 | except: |
|
19 | except: | |
| 19 | from time import sleep |
|
20 | from time import sleep | |
| 20 |
|
21 | |||
| 21 | from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader |
|
22 | from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader | |
| 22 | from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width |
|
23 | from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width | |
| 23 |
|
24 | |||
| 24 | LOCALTIME = True |
|
25 | LOCALTIME = True | |
| 25 |
|
26 | |||
| 26 | def isNumber(cad): |
|
27 | def isNumber(cad): | |
| 27 | """ |
|
28 | """ | |
| 28 | Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero. |
|
29 | Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero. | |
| 29 |
|
30 | |||
| 30 | Excepciones: |
|
31 | Excepciones: | |
| 31 | Si un determinado string no puede ser convertido a numero |
|
32 | Si un determinado string no puede ser convertido a numero | |
| 32 | Input: |
|
33 | Input: | |
| 33 | str, string al cual se le analiza para determinar si convertible a un numero o no |
|
34 | str, string al cual se le analiza para determinar si convertible a un numero o no | |
| 34 |
|
35 | |||
| 35 | Return: |
|
36 | Return: | |
| 36 | True : si el string es uno numerico |
|
37 | True : si el string es uno numerico | |
| 37 | False : no es un string numerico |
|
38 | False : no es un string numerico | |
| 38 | """ |
|
39 | """ | |
| 39 | try: |
|
40 | try: | |
| 40 | float( cad ) |
|
41 | float( cad ) | |
| 41 | return True |
|
42 | return True | |
| 42 | except: |
|
43 | except: | |
| 43 | return False |
|
44 | return False | |
| 44 |
|
45 | |||
| 45 | def isFileInEpoch(filename, startUTSeconds, endUTSeconds): |
|
46 | def isFileInEpoch(filename, startUTSeconds, endUTSeconds): | |
| 46 | """ |
|
47 | """ | |
| 47 | Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado. |
|
48 | Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado. | |
| 48 |
|
49 | |||
| 49 | Inputs: |
|
50 | Inputs: | |
| 50 | filename : nombre completo del archivo de datos en formato Jicamarca (.r) |
|
51 | filename : nombre completo del archivo de datos en formato Jicamarca (.r) | |
| 51 |
|
52 | |||
| 52 | startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en |
|
53 | startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en | |
| 53 | segundos contados desde 01/01/1970. |
|
54 | segundos contados desde 01/01/1970. | |
| 54 | endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en |
|
55 | endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en | |
| 55 | segundos contados desde 01/01/1970. |
|
56 | segundos contados desde 01/01/1970. | |
| 56 |
|
57 | |||
| 57 | Return: |
|
58 | Return: | |
| 58 | Boolean : Retorna True si el archivo de datos contiene datos en el rango de |
|
59 | Boolean : Retorna True si el archivo de datos contiene datos en el rango de | |
| 59 | fecha especificado, de lo contrario retorna False. |
|
60 | fecha especificado, de lo contrario retorna False. | |
| 60 |
|
61 | |||
| 61 | Excepciones: |
|
62 | Excepciones: | |
| 62 | Si el archivo no existe o no puede ser abierto |
|
63 | Si el archivo no existe o no puede ser abierto | |
| 63 | Si la cabecera no puede ser leida. |
|
64 | Si la cabecera no puede ser leida. | |
| 64 |
|
65 | |||
| 65 | """ |
|
66 | """ | |
| 66 | basicHeaderObj = BasicHeader(LOCALTIME) |
|
67 | basicHeaderObj = BasicHeader(LOCALTIME) | |
| 67 |
|
68 | |||
| 68 | try: |
|
69 | try: | |
| 69 | fp = open(filename,'rb') |
|
70 | fp = open(filename,'rb') | |
| 70 | except IOError: |
|
71 | except IOError: | |
| 71 | print "The file %s can't be opened" %(filename) |
|
72 | print "The file %s can't be opened" %(filename) | |
| 72 | return 0 |
|
73 | return 0 | |
| 73 |
|
74 | |||
| 74 | sts = basicHeaderObj.read(fp) |
|
75 | sts = basicHeaderObj.read(fp) | |
| 75 | fp.close() |
|
76 | fp.close() | |
| 76 |
|
77 | |||
| 77 | if not(sts): |
|
78 | if not(sts): | |
| 78 | print "Skipping the file %s because it has not a valid header" %(filename) |
|
79 | print "Skipping the file %s because it has not a valid header" %(filename) | |
| 79 | return 0 |
|
80 | return 0 | |
| 80 |
|
81 | |||
| 81 | if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)): |
|
82 | if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)): | |
| 82 | return 0 |
|
83 | return 0 | |
| 83 |
|
84 | |||
| 84 | return 1 |
|
85 | return 1 | |
| 85 |
|
86 | |||
| 86 | def isTimeInRange(thisTime, startTime, endTime): |
|
87 | def isTimeInRange(thisTime, startTime, endTime): | |
| 87 |
|
88 | |||
| 88 | if endTime >= startTime: |
|
89 | if endTime >= startTime: | |
| 89 | if (thisTime < startTime) or (thisTime > endTime): |
|
90 | if (thisTime < startTime) or (thisTime > endTime): | |
| 90 | return 0 |
|
91 | return 0 | |
| 91 |
|
92 | |||
| 92 | return 1 |
|
93 | return 1 | |
| 93 | else: |
|
94 | else: | |
| 94 | if (thisTime < startTime) and (thisTime > endTime): |
|
95 | if (thisTime < startTime) and (thisTime > endTime): | |
| 95 | return 0 |
|
96 | return 0 | |
| 96 |
|
97 | |||
| 97 | return 1 |
|
98 | return 1 | |
| 98 |
|
99 | |||
| 99 | def isFileInTimeRange(filename, startDate, endDate, startTime, endTime): |
|
100 | def isFileInTimeRange(filename, startDate, endDate, startTime, endTime): | |
| 100 | """ |
|
101 | """ | |
| 101 | Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. |
|
102 | Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. | |
| 102 |
|
103 | |||
| 103 | Inputs: |
|
104 | Inputs: | |
| 104 | filename : nombre completo del archivo de datos en formato Jicamarca (.r) |
|
105 | filename : nombre completo del archivo de datos en formato Jicamarca (.r) | |
| 105 |
|
106 | |||
| 106 | startDate : fecha inicial del rango seleccionado en formato datetime.date |
|
107 | startDate : fecha inicial del rango seleccionado en formato datetime.date | |
| 107 |
|
108 | |||
| 108 | endDate : fecha final del rango seleccionado en formato datetime.date |
|
109 | endDate : fecha final del rango seleccionado en formato datetime.date | |
| 109 |
|
110 | |||
| 110 | startTime : tiempo inicial del rango seleccionado en formato datetime.time |
|
111 | startTime : tiempo inicial del rango seleccionado en formato datetime.time | |
| 111 |
|
112 | |||
| 112 | endTime : tiempo final del rango seleccionado en formato datetime.time |
|
113 | endTime : tiempo final del rango seleccionado en formato datetime.time | |
| 113 |
|
114 | |||
| 114 | Return: |
|
115 | Return: | |
| 115 | Boolean : Retorna True si el archivo de datos contiene datos en el rango de |
|
116 | Boolean : Retorna True si el archivo de datos contiene datos en el rango de | |
| 116 | fecha especificado, de lo contrario retorna False. |
|
117 | fecha especificado, de lo contrario retorna False. | |
| 117 |
|
118 | |||
| 118 | Excepciones: |
|
119 | Excepciones: | |
| 119 | Si el archivo no existe o no puede ser abierto |
|
120 | Si el archivo no existe o no puede ser abierto | |
| 120 | Si la cabecera no puede ser leida. |
|
121 | Si la cabecera no puede ser leida. | |
| 121 |
|
122 | |||
| 122 | """ |
|
123 | """ | |
| 123 |
|
124 | |||
| 124 |
|
125 | |||
| 125 | try: |
|
126 | try: | |
| 126 | fp = open(filename,'rb') |
|
127 | fp = open(filename,'rb') | |
| 127 | except IOError: |
|
128 | except IOError: | |
| 128 | print "The file %s can't be opened" %(filename) |
|
129 | print "The file %s can't be opened" %(filename) | |
| 129 | return None |
|
130 | return None | |
| 130 |
|
131 | |||
| 131 | firstBasicHeaderObj = BasicHeader(LOCALTIME) |
|
132 | firstBasicHeaderObj = BasicHeader(LOCALTIME) | |
| 132 | systemHeaderObj = SystemHeader() |
|
133 | systemHeaderObj = SystemHeader() | |
| 133 | radarControllerHeaderObj = RadarControllerHeader() |
|
134 | radarControllerHeaderObj = RadarControllerHeader() | |
| 134 | processingHeaderObj = ProcessingHeader() |
|
135 | processingHeaderObj = ProcessingHeader() | |
| 135 |
|
136 | |||
| 136 | lastBasicHeaderObj = BasicHeader(LOCALTIME) |
|
137 | lastBasicHeaderObj = BasicHeader(LOCALTIME) | |
| 137 |
|
138 | |||
| 138 | sts = firstBasicHeaderObj.read(fp) |
|
139 | sts = firstBasicHeaderObj.read(fp) | |
| 139 |
|
140 | |||
| 140 | if not(sts): |
|
141 | if not(sts): | |
| 141 | print "[Reading] Skipping the file %s because it has not a valid header" %(filename) |
|
142 | print "[Reading] Skipping the file %s because it has not a valid header" %(filename) | |
| 142 | return None |
|
143 | return None | |
| 143 |
|
144 | |||
| 144 | if not systemHeaderObj.read(fp): |
|
145 | if not systemHeaderObj.read(fp): | |
| 145 | return None |
|
146 | return None | |
| 146 |
|
147 | |||
| 147 | if not radarControllerHeaderObj.read(fp): |
|
148 | if not radarControllerHeaderObj.read(fp): | |
| 148 | return None |
|
149 | return None | |
| 149 |
|
150 | |||
| 150 | if not processingHeaderObj.read(fp): |
|
151 | if not processingHeaderObj.read(fp): | |
| 151 | return None |
|
152 | return None | |
| 152 |
|
153 | |||
| 153 | filesize = os.path.getsize(filename) |
|
154 | filesize = os.path.getsize(filename) | |
| 154 |
|
155 | |||
| 155 | offset = processingHeaderObj.blockSize + 24 #header size |
|
156 | offset = processingHeaderObj.blockSize + 24 #header size | |
| 156 |
|
157 | |||
| 157 | if filesize <= offset: |
|
158 | if filesize <= offset: | |
| 158 | print "[Reading] %s: This file has not enough data" %filename |
|
159 | print "[Reading] %s: This file has not enough data" %filename | |
| 159 | return None |
|
160 | return None | |
| 160 |
|
161 | |||
| 161 | fp.seek(-offset, 2) |
|
162 | fp.seek(-offset, 2) | |
| 162 |
|
163 | |||
| 163 | sts = lastBasicHeaderObj.read(fp) |
|
164 | sts = lastBasicHeaderObj.read(fp) | |
| 164 |
|
165 | |||
| 165 | fp.close() |
|
166 | fp.close() | |
| 166 |
|
167 | |||
| 167 | thisDatetime = lastBasicHeaderObj.datatime |
|
168 | thisDatetime = lastBasicHeaderObj.datatime | |
| 168 | thisTime_last_block = thisDatetime.time() |
|
169 | thisTime_last_block = thisDatetime.time() | |
| 169 |
|
170 | |||
| 170 | thisDatetime = firstBasicHeaderObj.datatime |
|
171 | thisDatetime = firstBasicHeaderObj.datatime | |
| 171 | thisDate = thisDatetime.date() |
|
172 | thisDate = thisDatetime.date() | |
| 172 | thisTime_first_block = thisDatetime.time() |
|
173 | thisTime_first_block = thisDatetime.time() | |
| 173 |
|
174 | |||
| 174 | #General case |
|
175 | #General case | |
| 175 | # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o |
|
176 | # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o | |
| 176 | #-----------o----------------------------o----------- |
|
177 | #-----------o----------------------------o----------- | |
| 177 | # startTime endTime |
|
178 | # startTime endTime | |
| 178 |
|
179 | |||
| 179 | if endTime >= startTime: |
|
180 | if endTime >= startTime: | |
| 180 | if (thisTime_last_block < startTime) or (thisTime_first_block > endTime): |
|
181 | if (thisTime_last_block < startTime) or (thisTime_first_block > endTime): | |
| 181 | return None |
|
182 | return None | |
| 182 |
|
183 | |||
| 183 | return thisDatetime |
|
184 | return thisDatetime | |
| 184 |
|
185 | |||
| 185 | #If endTime < startTime then endTime belongs to the next day |
|
186 | #If endTime < startTime then endTime belongs to the next day | |
| 186 |
|
187 | |||
| 187 |
|
188 | |||
| 188 | #<<<<<<<<<<<o o>>>>>>>>>>> |
|
189 | #<<<<<<<<<<<o o>>>>>>>>>>> | |
| 189 | #-----------o----------------------------o----------- |
|
190 | #-----------o----------------------------o----------- | |
| 190 | # endTime startTime |
|
191 | # endTime startTime | |
| 191 |
|
192 | |||
| 192 | if (thisDate == startDate) and (thisTime_last_block < startTime): |
|
193 | if (thisDate == startDate) and (thisTime_last_block < startTime): | |
| 193 | return None |
|
194 | return None | |
| 194 |
|
195 | |||
| 195 | if (thisDate == endDate) and (thisTime_first_block > endTime): |
|
196 | if (thisDate == endDate) and (thisTime_first_block > endTime): | |
| 196 | return None |
|
197 | return None | |
| 197 |
|
198 | |||
| 198 | if (thisTime_last_block < startTime) and (thisTime_first_block > endTime): |
|
199 | if (thisTime_last_block < startTime) and (thisTime_first_block > endTime): | |
| 199 | return None |
|
200 | return None | |
| 200 |
|
201 | |||
| 201 | return thisDatetime |
|
202 | return thisDatetime | |
| 202 |
|
203 | |||
| 203 | def isFolderInDateRange(folder, startDate=None, endDate=None): |
|
204 | def isFolderInDateRange(folder, startDate=None, endDate=None): | |
| 204 | """ |
|
205 | """ | |
| 205 | Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. |
|
206 | Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. | |
| 206 |
|
207 | |||
| 207 | Inputs: |
|
208 | Inputs: | |
| 208 | folder : nombre completo del directorio. |
|
209 | folder : nombre completo del directorio. | |
| 209 | Su formato deberia ser "/path_root/?YYYYDDD" |
|
210 | Su formato deberia ser "/path_root/?YYYYDDD" | |
| 210 |
|
211 | |||
| 211 | siendo: |
|
212 | siendo: | |
| 212 | YYYY : Anio (ejemplo 2015) |
|
213 | YYYY : Anio (ejemplo 2015) | |
| 213 | DDD : Dia del anio (ejemplo 305) |
|
214 | DDD : Dia del anio (ejemplo 305) | |
| 214 |
|
215 | |||
| 215 | startDate : fecha inicial del rango seleccionado en formato datetime.date |
|
216 | startDate : fecha inicial del rango seleccionado en formato datetime.date | |
| 216 |
|
217 | |||
| 217 | endDate : fecha final del rango seleccionado en formato datetime.date |
|
218 | endDate : fecha final del rango seleccionado en formato datetime.date | |
| 218 |
|
219 | |||
| 219 | Return: |
|
220 | Return: | |
| 220 | Boolean : Retorna True si el archivo de datos contiene datos en el rango de |
|
221 | Boolean : Retorna True si el archivo de datos contiene datos en el rango de | |
| 221 | fecha especificado, de lo contrario retorna False. |
|
222 | fecha especificado, de lo contrario retorna False. | |
| 222 | Excepciones: |
|
223 | Excepciones: | |
| 223 | Si el directorio no tiene el formato adecuado |
|
224 | Si el directorio no tiene el formato adecuado | |
| 224 | """ |
|
225 | """ | |
| 225 |
|
226 | |||
| 226 | basename = os.path.basename(folder) |
|
227 | basename = os.path.basename(folder) | |
| 227 |
|
228 | |||
| 228 | if not isRadarFolder(basename): |
|
229 | if not isRadarFolder(basename): | |
| 229 | print "The folder %s has not the rigth format" %folder |
|
230 | print "The folder %s has not the rigth format" %folder | |
| 230 | return 0 |
|
231 | return 0 | |
| 231 |
|
232 | |||
| 232 | if startDate and endDate: |
|
233 | if startDate and endDate: | |
| 233 | thisDate = getDateFromRadarFolder(basename) |
|
234 | thisDate = getDateFromRadarFolder(basename) | |
| 234 |
|
235 | |||
| 235 | if thisDate < startDate: |
|
236 | if thisDate < startDate: | |
| 236 | return 0 |
|
237 | return 0 | |
| 237 |
|
238 | |||
| 238 | if thisDate > endDate: |
|
239 | if thisDate > endDate: | |
| 239 | return 0 |
|
240 | return 0 | |
| 240 |
|
241 | |||
| 241 | return 1 |
|
242 | return 1 | |
| 242 |
|
243 | |||
| 243 | def isFileInDateRange(filename, startDate=None, endDate=None): |
|
244 | def isFileInDateRange(filename, startDate=None, endDate=None): | |
| 244 | """ |
|
245 | """ | |
| 245 | Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. |
|
246 | Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado. | |
| 246 |
|
247 | |||
| 247 | Inputs: |
|
248 | Inputs: | |
| 248 | filename : nombre completo del archivo de datos en formato Jicamarca (.r) |
|
249 | filename : nombre completo del archivo de datos en formato Jicamarca (.r) | |
| 249 |
|
250 | |||
| 250 | Su formato deberia ser "?YYYYDDDsss" |
|
251 | Su formato deberia ser "?YYYYDDDsss" | |
| 251 |
|
252 | |||
| 252 | siendo: |
|
253 | siendo: | |
| 253 | YYYY : Anio (ejemplo 2015) |
|
254 | YYYY : Anio (ejemplo 2015) | |
| 254 | DDD : Dia del anio (ejemplo 305) |
|
255 | DDD : Dia del anio (ejemplo 305) | |
| 255 | sss : set |
|
256 | sss : set | |
| 256 |
|
257 | |||
| 257 | startDate : fecha inicial del rango seleccionado en formato datetime.date |
|
258 | startDate : fecha inicial del rango seleccionado en formato datetime.date | |
| 258 |
|
259 | |||
| 259 | endDate : fecha final del rango seleccionado en formato datetime.date |
|
260 | endDate : fecha final del rango seleccionado en formato datetime.date | |
| 260 |
|
261 | |||
| 261 | Return: |
|
262 | Return: | |
| 262 | Boolean : Retorna True si el archivo de datos contiene datos en el rango de |
|
263 | Boolean : Retorna True si el archivo de datos contiene datos en el rango de | |
| 263 | fecha especificado, de lo contrario retorna False. |
|
264 | fecha especificado, de lo contrario retorna False. | |
| 264 | Excepciones: |
|
265 | Excepciones: | |
| 265 | Si el archivo no tiene el formato adecuado |
|
266 | Si el archivo no tiene el formato adecuado | |
| 266 | """ |
|
267 | """ | |
| 267 |
|
268 | |||
| 268 | basename = os.path.basename(filename) |
|
269 | basename = os.path.basename(filename) | |
| 269 |
|
270 | |||
| 270 | if not isRadarFile(basename): |
|
271 | if not isRadarFile(basename): | |
| 271 | print "The filename %s has not the rigth format" %filename |
|
272 | print "The filename %s has not the rigth format" %filename | |
| 272 | return 0 |
|
273 | return 0 | |
| 273 |
|
274 | |||
| 274 | if startDate and endDate: |
|
275 | if startDate and endDate: | |
| 275 | thisDate = getDateFromRadarFile(basename) |
|
276 | thisDate = getDateFromRadarFile(basename) | |
| 276 |
|
277 | |||
| 277 | if thisDate < startDate: |
|
278 | if thisDate < startDate: | |
| 278 | return 0 |
|
279 | return 0 | |
| 279 |
|
280 | |||
| 280 | if thisDate > endDate: |
|
281 | if thisDate > endDate: | |
| 281 | return 0 |
|
282 | return 0 | |
| 282 |
|
283 | |||
| 283 | return 1 |
|
284 | return 1 | |
| 284 |
|
285 | |||
| 285 | def getFileFromSet(path, ext, set): |
|
286 | def getFileFromSet(path, ext, set): | |
| 286 | validFilelist = [] |
|
287 | validFilelist = [] | |
| 287 | fileList = os.listdir(path) |
|
288 | fileList = os.listdir(path) | |
| 288 |
|
289 | |||
| 289 | # 0 1234 567 89A BCDE |
|
290 | # 0 1234 567 89A BCDE | |
| 290 | # H YYYY DDD SSS .ext |
|
291 | # H YYYY DDD SSS .ext | |
| 291 |
|
292 | |||
| 292 | for thisFile in fileList: |
|
293 | for thisFile in fileList: | |
| 293 | try: |
|
294 | try: | |
| 294 | year = int(thisFile[1:5]) |
|
295 | year = int(thisFile[1:5]) | |
| 295 | doy = int(thisFile[5:8]) |
|
296 | doy = int(thisFile[5:8]) | |
| 296 | except: |
|
297 | except: | |
| 297 | continue |
|
298 | continue | |
| 298 |
|
299 | |||
| 299 | if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): |
|
300 | if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): | |
| 300 | continue |
|
301 | continue | |
| 301 |
|
302 | |||
| 302 | validFilelist.append(thisFile) |
|
303 | validFilelist.append(thisFile) | |
| 303 |
|
304 | |||
| 304 | myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set)) |
|
305 | myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set)) | |
| 305 |
|
306 | |||
| 306 | if len(myfile)!= 0: |
|
307 | if len(myfile)!= 0: | |
| 307 | return myfile[0] |
|
308 | return myfile[0] | |
| 308 | else: |
|
309 | else: | |
| 309 | filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower()) |
|
310 | filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower()) | |
| 310 | print 'the filename %s does not exist'%filename |
|
311 | print 'the filename %s does not exist'%filename | |
| 311 | print '...going to the last file: ' |
|
312 | print '...going to the last file: ' | |
| 312 |
|
313 | |||
| 313 | if validFilelist: |
|
314 | if validFilelist: | |
| 314 | validFilelist = sorted( validFilelist, key=str.lower ) |
|
315 | validFilelist = sorted( validFilelist, key=str.lower ) | |
| 315 | return validFilelist[-1] |
|
316 | return validFilelist[-1] | |
| 316 |
|
317 | |||
| 317 | return None |
|
318 | return None | |
| 318 |
|
319 | |||
| 319 | def getlastFileFromPath(path, ext): |
|
320 | def getlastFileFromPath(path, ext): | |
| 320 | """ |
|
321 | """ | |
| 321 | Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext" |
|
322 | Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext" | |
| 322 | al final de la depuracion devuelve el ultimo file de la lista que quedo. |
|
323 | al final de la depuracion devuelve el ultimo file de la lista que quedo. | |
| 323 |
|
324 | |||
| 324 | Input: |
|
325 | Input: | |
| 325 | fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta |
|
326 | fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta | |
| 326 | ext : extension de los files contenidos en una carpeta |
|
327 | ext : extension de los files contenidos en una carpeta | |
| 327 |
|
328 | |||
| 328 | Return: |
|
329 | Return: | |
| 329 | El ultimo file de una determinada carpeta, no se considera el path. |
|
330 | El ultimo file de una determinada carpeta, no se considera el path. | |
| 330 | """ |
|
331 | """ | |
| 331 | validFilelist = [] |
|
332 | validFilelist = [] | |
| 332 | fileList = os.listdir(path) |
|
333 | fileList = os.listdir(path) | |
| 333 |
|
334 | |||
| 334 | # 0 1234 567 89A BCDE |
|
335 | # 0 1234 567 89A BCDE | |
| 335 | # H YYYY DDD SSS .ext |
|
336 | # H YYYY DDD SSS .ext | |
| 336 |
|
337 | |||
| 337 | for thisFile in fileList: |
|
338 | for thisFile in fileList: | |
| 338 |
|
339 | |||
| 339 | year = thisFile[1:5] |
|
340 | year = thisFile[1:5] | |
| 340 | if not isNumber(year): |
|
341 | if not isNumber(year): | |
| 341 | continue |
|
342 | continue | |
| 342 |
|
343 | |||
| 343 | doy = thisFile[5:8] |
|
344 | doy = thisFile[5:8] | |
| 344 | if not isNumber(doy): |
|
345 | if not isNumber(doy): | |
| 345 | continue |
|
346 | continue | |
| 346 |
|
347 | |||
| 347 | year = int(year) |
|
348 | year = int(year) | |
| 348 | doy = int(doy) |
|
349 | doy = int(doy) | |
| 349 |
|
350 | |||
| 350 | if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): |
|
351 | if (os.path.splitext(thisFile)[-1].lower() != ext.lower()): | |
| 351 | continue |
|
352 | continue | |
| 352 |
|
353 | |||
| 353 | validFilelist.append(thisFile) |
|
354 | validFilelist.append(thisFile) | |
| 354 |
|
355 | |||
| 355 | if validFilelist: |
|
356 | if validFilelist: | |
| 356 | validFilelist = sorted( validFilelist, key=str.lower ) |
|
357 | validFilelist = sorted( validFilelist, key=str.lower ) | |
| 357 | return validFilelist[-1] |
|
358 | return validFilelist[-1] | |
| 358 |
|
359 | |||
| 359 | return None |
|
360 | return None | |
| 360 |
|
361 | |||
| 361 | def checkForRealPath(path, foldercounter, year, doy, set, ext): |
|
362 | def checkForRealPath(path, foldercounter, year, doy, set, ext): | |
| 362 | """ |
|
363 | """ | |
| 363 | Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path, |
|
364 | Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path, | |
| 364 | Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar |
|
365 | Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar | |
| 365 | el path exacto de un determinado file. |
|
366 | el path exacto de un determinado file. | |
| 366 |
|
367 | |||
| 367 | Example : |
|
368 | Example : | |
| 368 | nombre correcto del file es .../.../D2009307/P2009307367.ext |
|
369 | nombre correcto del file es .../.../D2009307/P2009307367.ext | |
| 369 |
|
370 | |||
| 370 | Entonces la funcion prueba con las siguientes combinaciones |
|
371 | Entonces la funcion prueba con las siguientes combinaciones | |
| 371 | .../.../y2009307367.ext |
|
372 | .../.../y2009307367.ext | |
| 372 | .../.../Y2009307367.ext |
|
373 | .../.../Y2009307367.ext | |
| 373 | .../.../x2009307/y2009307367.ext |
|
374 | .../.../x2009307/y2009307367.ext | |
| 374 | .../.../x2009307/Y2009307367.ext |
|
375 | .../.../x2009307/Y2009307367.ext | |
| 375 | .../.../X2009307/y2009307367.ext |
|
376 | .../.../X2009307/y2009307367.ext | |
| 376 | .../.../X2009307/Y2009307367.ext |
|
377 | .../.../X2009307/Y2009307367.ext | |
| 377 | siendo para este caso, la ultima combinacion de letras, identica al file buscado |
|
378 | siendo para este caso, la ultima combinacion de letras, identica al file buscado | |
| 378 |
|
379 | |||
| 379 | Return: |
|
380 | Return: | |
| 380 | Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file |
|
381 | Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file | |
| 381 | caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas |
|
382 | caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas | |
| 382 | para el filename |
|
383 | para el filename | |
| 383 | """ |
|
384 | """ | |
| 384 | fullfilename = None |
|
385 | fullfilename = None | |
| 385 | find_flag = False |
|
386 | find_flag = False | |
| 386 | filename = None |
|
387 | filename = None | |
| 387 |
|
388 | |||
| 388 | prefixDirList = [None,'d','D'] |
|
389 | prefixDirList = [None,'d','D'] | |
| 389 | if ext.lower() == ".r": #voltage |
|
390 | if ext.lower() == ".r": #voltage | |
| 390 | prefixFileList = ['d','D'] |
|
391 | prefixFileList = ['d','D'] | |
| 391 | elif ext.lower() == ".pdata": #spectra |
|
392 | elif ext.lower() == ".pdata": #spectra | |
| 392 | prefixFileList = ['p','P'] |
|
393 | prefixFileList = ['p','P'] | |
| 393 | else: |
|
394 | else: | |
| 394 | return None, filename |
|
395 | return None, filename | |
| 395 |
|
396 | |||
| 396 | #barrido por las combinaciones posibles |
|
397 | #barrido por las combinaciones posibles | |
| 397 | for prefixDir in prefixDirList: |
|
398 | for prefixDir in prefixDirList: | |
| 398 | thispath = path |
|
399 | thispath = path | |
| 399 | if prefixDir != None: |
|
400 | if prefixDir != None: | |
| 400 | #formo el nombre del directorio xYYYYDDD (x=d o x=D) |
|
401 | #formo el nombre del directorio xYYYYDDD (x=d o x=D) | |
| 401 | if foldercounter == 0: |
|
402 | if foldercounter == 0: | |
| 402 | thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy )) |
|
403 | thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy )) | |
| 403 | else: |
|
404 | else: | |
| 404 | thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter)) |
|
405 | thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter)) | |
| 405 | for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D" |
|
406 | for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D" | |
| 406 | filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext |
|
407 | filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext | |
| 407 | fullfilename = os.path.join( thispath, filename ) #formo el path completo |
|
408 | fullfilename = os.path.join( thispath, filename ) #formo el path completo | |
| 408 |
|
409 | |||
| 409 | if os.path.exists( fullfilename ): #verifico que exista |
|
410 | if os.path.exists( fullfilename ): #verifico que exista | |
| 410 | find_flag = True |
|
411 | find_flag = True | |
| 411 | break |
|
412 | break | |
| 412 | if find_flag: |
|
413 | if find_flag: | |
| 413 | break |
|
414 | break | |
| 414 |
|
415 | |||
| 415 | if not(find_flag): |
|
416 | if not(find_flag): | |
| 416 | return None, filename |
|
417 | return None, filename | |
| 417 |
|
418 | |||
| 418 | return fullfilename, filename |
|
419 | return fullfilename, filename | |
| 419 |
|
420 | |||
| 420 | def isRadarFolder(folder): |
|
421 | def isRadarFolder(folder): | |
| 421 | try: |
|
422 | try: | |
| 422 | year = int(folder[1:5]) |
|
423 | year = int(folder[1:5]) | |
| 423 | doy = int(folder[5:8]) |
|
424 | doy = int(folder[5:8]) | |
| 424 | except: |
|
425 | except: | |
| 425 | return 0 |
|
426 | return 0 | |
| 426 |
|
427 | |||
| 427 | return 1 |
|
428 | return 1 | |
| 428 |
|
429 | |||
| 429 | def isRadarFile(file): |
|
430 | def isRadarFile(file): | |
| 430 | try: |
|
431 | try: | |
| 431 | year = int(file[1:5]) |
|
432 | year = int(file[1:5]) | |
| 432 | doy = int(file[5:8]) |
|
433 | doy = int(file[5:8]) | |
| 433 | set = int(file[8:11]) |
|
434 | set = int(file[8:11]) | |
| 434 | except: |
|
435 | except: | |
| 435 | return 0 |
|
436 | return 0 | |
| 436 |
|
437 | |||
| 437 | return 1 |
|
438 | return 1 | |
| 438 |
|
439 | |||
| 439 | def getDateFromRadarFile(file): |
|
440 | def getDateFromRadarFile(file): | |
| 440 | try: |
|
441 | try: | |
| 441 | year = int(file[1:5]) |
|
442 | year = int(file[1:5]) | |
| 442 | doy = int(file[5:8]) |
|
443 | doy = int(file[5:8]) | |
| 443 | set = int(file[8:11]) |
|
444 | set = int(file[8:11]) | |
| 444 | except: |
|
445 | except: | |
| 445 | return None |
|
446 | return None | |
| 446 |
|
447 | |||
| 447 | thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1) |
|
448 | thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1) | |
| 448 | return thisDate |
|
449 | return thisDate | |
| 449 |
|
450 | |||
| 450 | def getDateFromRadarFolder(folder): |
|
451 | def getDateFromRadarFolder(folder): | |
| 451 | try: |
|
452 | try: | |
| 452 | year = int(folder[1:5]) |
|
453 | year = int(folder[1:5]) | |
| 453 | doy = int(folder[5:8]) |
|
454 | doy = int(folder[5:8]) | |
| 454 | except: |
|
455 | except: | |
| 455 | return None |
|
456 | return None | |
| 456 |
|
457 | |||
| 457 | thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1) |
|
458 | thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1) | |
| 458 | return thisDate |
|
459 | return thisDate | |
| 459 |
|
460 | |||
| 460 | class JRODataIO: |
|
461 | class JRODataIO: | |
| 461 |
|
462 | |||
| 462 | c = 3E8 |
|
463 | c = 3E8 | |
| 463 |
|
464 | |||
| 464 | isConfig = False |
|
465 | isConfig = False | |
| 465 |
|
466 | |||
| 466 | basicHeaderObj = None |
|
467 | basicHeaderObj = None | |
| 467 |
|
468 | |||
| 468 | systemHeaderObj = None |
|
469 | systemHeaderObj = None | |
| 469 |
|
470 | |||
| 470 | radarControllerHeaderObj = None |
|
471 | radarControllerHeaderObj = None | |
| 471 |
|
472 | |||
| 472 | processingHeaderObj = None |
|
473 | processingHeaderObj = None | |
| 473 |
|
474 | |||
| 474 | dtype = None |
|
475 | dtype = None | |
| 475 |
|
476 | |||
| 476 | pathList = [] |
|
477 | pathList = [] | |
| 477 |
|
478 | |||
| 478 | filenameList = [] |
|
479 | filenameList = [] | |
| 479 |
|
480 | |||
| 480 | filename = None |
|
481 | filename = None | |
| 481 |
|
482 | |||
| 482 | ext = None |
|
483 | ext = None | |
| 483 |
|
484 | |||
| 484 | flagIsNewFile = 1 |
|
485 | flagIsNewFile = 1 | |
| 485 |
|
486 | |||
| 486 | flagDiscontinuousBlock = 0 |
|
487 | flagDiscontinuousBlock = 0 | |
| 487 |
|
488 | |||
| 488 | flagIsNewBlock = 0 |
|
489 | flagIsNewBlock = 0 | |
| 489 |
|
490 | |||
| 490 | fp = None |
|
491 | fp = None | |
| 491 |
|
492 | |||
| 492 | firstHeaderSize = 0 |
|
493 | firstHeaderSize = 0 | |
| 493 |
|
494 | |||
| 494 | basicHeaderSize = 24 |
|
495 | basicHeaderSize = 24 | |
| 495 |
|
496 | |||
| 496 | versionFile = 1103 |
|
497 | versionFile = 1103 | |
| 497 |
|
498 | |||
| 498 | fileSize = None |
|
499 | fileSize = None | |
| 499 |
|
500 | |||
| 500 | # ippSeconds = None |
|
501 | # ippSeconds = None | |
| 501 |
|
502 | |||
| 502 | fileSizeByHeader = None |
|
503 | fileSizeByHeader = None | |
| 503 |
|
504 | |||
| 504 | fileIndex = None |
|
505 | fileIndex = None | |
| 505 |
|
506 | |||
| 506 | profileIndex = None |
|
507 | profileIndex = None | |
| 507 |
|
508 | |||
| 508 | blockIndex = None |
|
509 | blockIndex = None | |
| 509 |
|
510 | |||
| 510 | nTotalBlocks = None |
|
511 | nTotalBlocks = None | |
| 511 |
|
512 | |||
| 512 | maxTimeStep = 30 |
|
513 | maxTimeStep = 30 | |
| 513 |
|
514 | |||
| 514 | lastUTTime = None |
|
515 | lastUTTime = None | |
| 515 |
|
516 | |||
| 516 | datablock = None |
|
517 | datablock = None | |
| 517 |
|
518 | |||
| 518 | dataOut = None |
|
519 | dataOut = None | |
| 519 |
|
520 | |||
| 520 | blocksize = None |
|
521 | blocksize = None | |
| 521 |
|
522 | |||
| 522 | getByBlock = False |
|
523 | getByBlock = False | |
| 523 |
|
524 | |||
| 524 | def __init__(self): |
|
525 | def __init__(self): | |
| 525 |
|
526 | |||
| 526 | raise NotImplementedError |
|
527 | raise NotImplementedError | |
| 527 |
|
528 | |||
| 528 | def run(self): |
|
529 | def run(self): | |
| 529 |
|
530 | |||
| 530 | raise NotImplementedError |
|
531 | raise NotImplementedError | |
| 531 |
|
532 | |||
| 532 | def getDtypeWidth(self): |
|
533 | def getDtypeWidth(self): | |
| 533 |
|
534 | |||
| 534 | dtype_index = get_dtype_index(self.dtype) |
|
535 | dtype_index = get_dtype_index(self.dtype) | |
| 535 | dtype_width = get_dtype_width(dtype_index) |
|
536 | dtype_width = get_dtype_width(dtype_index) | |
| 536 |
|
537 | |||
| 537 | return dtype_width |
|
538 | return dtype_width | |
| 538 |
|
539 | |||
| 539 | def getAllowedArgs(self): |
|
540 | def getAllowedArgs(self): | |
| 540 | return inspect.getargspec(self.run).args |
|
541 | return inspect.getargspec(self.run).args | |
| 541 |
|
542 | |||
| 542 | class JRODataReader(JRODataIO): |
|
543 | class JRODataReader(JRODataIO): | |
| 543 |
|
544 | |||
| 544 |
|
545 | |||
| 545 | online = 0 |
|
546 | online = 0 | |
| 546 |
|
547 | |||
| 547 | realtime = 0 |
|
548 | realtime = 0 | |
| 548 |
|
549 | |||
| 549 | nReadBlocks = 0 |
|
550 | nReadBlocks = 0 | |
| 550 |
|
551 | |||
| 551 | delay = 10 #number of seconds waiting a new file |
|
552 | delay = 10 #number of seconds waiting a new file | |
| 552 |
|
553 | |||
| 553 | nTries = 3 #quantity tries |
|
554 | nTries = 3 #quantity tries | |
| 554 |
|
555 | |||
| 555 | nFiles = 3 #number of files for searching |
|
556 | nFiles = 3 #number of files for searching | |
| 556 |
|
557 | |||
| 557 | path = None |
|
558 | path = None | |
| 558 |
|
559 | |||
| 559 | foldercounter = 0 |
|
560 | foldercounter = 0 | |
| 560 |
|
561 | |||
| 561 | flagNoMoreFiles = 0 |
|
562 | flagNoMoreFiles = 0 | |
| 562 |
|
563 | |||
| 563 | datetimeList = [] |
|
564 | datetimeList = [] | |
| 564 |
|
565 | |||
| 565 | __isFirstTimeOnline = 1 |
|
566 | __isFirstTimeOnline = 1 | |
| 566 |
|
567 | |||
| 567 | __printInfo = True |
|
568 | __printInfo = True | |
| 568 |
|
569 | |||
| 569 | profileIndex = None |
|
570 | profileIndex = None | |
| 570 |
|
571 | |||
| 571 | nTxs = 1 |
|
572 | nTxs = 1 | |
| 572 |
|
573 | |||
| 573 | txIndex = None |
|
574 | txIndex = None | |
| 574 |
|
575 | |||
| 575 | #Added-------------------- |
|
576 | #Added-------------------- | |
| 576 |
|
577 | |||
| 577 | selBlocksize = None |
|
578 | selBlocksize = None | |
| 578 |
|
579 | |||
| 579 | selBlocktime = None |
|
580 | selBlocktime = None | |
| 580 |
|
581 | |||
| 581 |
|
582 | |||
| 582 | def __init__(self): |
|
583 | def __init__(self): | |
| 583 |
|
584 | |||
| 584 | """ |
|
585 | """ | |
| 585 | This class is used to find data files |
|
586 | This class is used to find data files | |
| 586 |
|
587 | |||
| 587 | Example: |
|
588 | Example: | |
| 588 | reader = JRODataReader() |
|
589 | reader = JRODataReader() | |
| 589 | fileList = reader.findDataFiles() |
|
590 | fileList = reader.findDataFiles() | |
| 590 |
|
591 | |||
| 591 | """ |
|
592 | """ | |
| 592 | pass |
|
593 | pass | |
| 593 |
|
594 | |||
| 594 |
|
595 | |||
| 595 | def createObjByDefault(self): |
|
596 | def createObjByDefault(self): | |
| 596 | """ |
|
597 | """ | |
| 597 |
|
598 | |||
| 598 | """ |
|
599 | """ | |
| 599 | raise NotImplementedError |
|
600 | raise NotImplementedError | |
| 600 |
|
601 | |||
| 601 | def getBlockDimension(self): |
|
602 | def getBlockDimension(self): | |
| 602 |
|
603 | |||
| 603 | raise NotImplementedError |
|
604 | raise NotImplementedError | |
| 604 |
|
605 | |||
| 605 | def __searchFilesOffLine(self, |
|
606 | def __searchFilesOffLine(self, | |
| 606 | path, |
|
607 | path, | |
| 607 | startDate=None, |
|
608 | startDate=None, | |
| 608 | endDate=None, |
|
609 | endDate=None, | |
| 609 | startTime=datetime.time(0,0,0), |
|
610 | startTime=datetime.time(0,0,0), | |
| 610 | endTime=datetime.time(23,59,59), |
|
611 | endTime=datetime.time(23,59,59), | |
| 611 | set=None, |
|
612 | set=None, | |
| 612 | expLabel='', |
|
613 | expLabel='', | |
| 613 | ext='.r', |
|
614 | ext='.r', | |
| 614 | queue=None, |
|
615 | queue=None, | |
| 615 | cursor=None, |
|
616 | cursor=None, | |
| 616 | skip=None, |
|
617 | skip=None, | |
| 617 | walk=True): |
|
618 | walk=True): | |
| 618 |
|
619 | |||
| 619 | self.filenameList = [] |
|
620 | self.filenameList = [] | |
| 620 | self.datetimeList = [] |
|
621 | self.datetimeList = [] | |
| 621 |
|
622 | |||
| 622 | pathList = [] |
|
623 | pathList = [] | |
| 623 |
|
624 | |||
| 624 | dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True) |
|
625 | dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True) | |
| 625 |
|
626 | |||
| 626 | if dateList == []: |
|
627 | if dateList == []: | |
| 627 | # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path) |
|
628 | # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path) | |
| 628 | return None, None |
|
629 | return None, None | |
| 629 |
|
630 | |||
| 630 | if len(dateList) > 1: |
|
631 | if len(dateList) > 1: | |
| 631 | print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList)) |
|
632 | print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList)) | |
| 632 | else: |
|
633 | else: | |
| 633 | print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0]) |
|
634 | print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0]) | |
| 634 |
|
635 | |||
| 635 | filenameList = [] |
|
636 | filenameList = [] | |
| 636 | datetimeList = [] |
|
637 | datetimeList = [] | |
| 637 |
|
638 | |||
| 638 | for thisPath in pathList: |
|
639 | for thisPath in pathList: | |
| 639 | # thisPath = pathList[pathDict[file]] |
|
640 | # thisPath = pathList[pathDict[file]] | |
| 640 |
|
641 | |||
| 641 | fileList = glob.glob1(thisPath, "*%s" %ext) |
|
642 | fileList = glob.glob1(thisPath, "*%s" %ext) | |
| 642 | fileList.sort() |
|
643 | fileList.sort() | |
| 643 |
|
644 | |||
| 644 | skippedFileList = [] |
|
645 | skippedFileList = [] | |
| 645 |
|
646 | |||
| 646 | if cursor is not None and skip is not None: |
|
647 | if cursor is not None and skip is not None: | |
| 647 | # if cursor*skip > len(fileList): |
|
648 | # if cursor*skip > len(fileList): | |
| 648 | if skip == 0: |
|
649 | if skip == 0: | |
| 649 | if queue is not None: |
|
650 | if queue is not None: | |
| 650 | queue.put(len(fileList)) |
|
651 | queue.put(len(fileList)) | |
| 651 | skippedFileList = [] |
|
652 | skippedFileList = [] | |
| 652 | else: |
|
653 | else: | |
| 653 | skippedFileList = fileList[cursor*skip: cursor*skip + skip] |
|
654 | skippedFileList = fileList[cursor*skip: cursor*skip + skip] | |
| 654 |
|
655 | |||
| 655 | else: |
|
656 | else: | |
| 656 | skippedFileList = fileList |
|
657 | skippedFileList = fileList | |
| 657 |
|
658 | |||
| 658 | for file in skippedFileList: |
|
659 | for file in skippedFileList: | |
| 659 |
|
660 | |||
| 660 | filename = os.path.join(thisPath,file) |
|
661 | filename = os.path.join(thisPath,file) | |
| 661 |
|
662 | |||
| 662 | if not isFileInDateRange(filename, startDate, endDate): |
|
663 | if not isFileInDateRange(filename, startDate, endDate): | |
| 663 | continue |
|
664 | continue | |
| 664 |
|
665 | |||
| 665 | thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime) |
|
666 | thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime) | |
| 666 |
|
667 | |||
| 667 | if not(thisDatetime): |
|
668 | if not(thisDatetime): | |
| 668 | continue |
|
669 | continue | |
| 669 |
|
670 | |||
| 670 | filenameList.append(filename) |
|
671 | filenameList.append(filename) | |
| 671 | datetimeList.append(thisDatetime) |
|
672 | datetimeList.append(thisDatetime) | |
| 672 |
|
673 | |||
| 673 | if not(filenameList): |
|
674 | if not(filenameList): | |
| 674 | print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path) |
|
675 | print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path) | |
| 675 | return None, None |
|
676 | return None, None | |
| 676 |
|
677 | |||
| 677 | print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime) |
|
678 | print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime) | |
| 678 |
|
679 | |||
| 679 |
|
680 | |||
| 680 | for i in range(len(filenameList)): |
|
681 | for i in range(len(filenameList)): | |
| 681 | print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) |
|
682 | print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) | |
| 682 |
|
683 | |||
| 683 | self.filenameList = filenameList |
|
684 | self.filenameList = filenameList | |
| 684 | self.datetimeList = datetimeList |
|
685 | self.datetimeList = datetimeList | |
| 685 |
|
686 | |||
| 686 | return pathList, filenameList |
|
687 | return pathList, filenameList | |
| 687 |
|
688 | |||
| 688 | def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None): |
|
689 | def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None): | |
| 689 |
|
690 | |||
| 690 | """ |
|
691 | """ | |
| 691 | Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y |
|
692 | Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y | |
| 692 | devuelve el archivo encontrado ademas de otros datos. |
|
693 | devuelve el archivo encontrado ademas de otros datos. | |
| 693 |
|
694 | |||
| 694 | Input: |
|
695 | Input: | |
| 695 | path : carpeta donde estan contenidos los files que contiene data |
|
696 | path : carpeta donde estan contenidos los files que contiene data | |
| 696 |
|
697 | |||
| 697 | expLabel : Nombre del subexperimento (subfolder) |
|
698 | expLabel : Nombre del subexperimento (subfolder) | |
| 698 |
|
699 | |||
| 699 | ext : extension de los files |
|
700 | ext : extension de los files | |
| 700 |
|
701 | |||
| 701 | walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath) |
|
702 | walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath) | |
| 702 |
|
703 | |||
| 703 | Return: |
|
704 | Return: | |
| 704 | directory : eL directorio donde esta el file encontrado |
|
705 | directory : eL directorio donde esta el file encontrado | |
| 705 | filename : el ultimo file de una determinada carpeta |
|
706 | filename : el ultimo file de una determinada carpeta | |
| 706 | year : el anho |
|
707 | year : el anho | |
| 707 | doy : el numero de dia del anho |
|
708 | doy : el numero de dia del anho | |
| 708 | set : el set del archivo |
|
709 | set : el set del archivo | |
| 709 |
|
710 | |||
| 710 |
|
711 | |||
| 711 | """ |
|
712 | """ | |
| 712 | if not os.path.isdir(path): |
|
713 | if not os.path.isdir(path): | |
| 713 | return None, None, None, None, None, None |
|
714 | return None, None, None, None, None, None | |
| 714 |
|
715 | |||
| 715 | dirList = [] |
|
716 | dirList = [] | |
| 716 |
|
717 | |||
| 717 | if not walk: |
|
718 | if not walk: | |
| 718 | fullpath = path |
|
719 | fullpath = path | |
| 719 | foldercounter = 0 |
|
720 | foldercounter = 0 | |
| 720 | else: |
|
721 | else: | |
| 721 | #Filtra solo los directorios |
|
722 | #Filtra solo los directorios | |
| 722 | for thisPath in os.listdir(path): |
|
723 | for thisPath in os.listdir(path): | |
| 723 | if not os.path.isdir(os.path.join(path,thisPath)): |
|
724 | if not os.path.isdir(os.path.join(path,thisPath)): | |
| 724 | continue |
|
725 | continue | |
| 725 | if not isRadarFolder(thisPath): |
|
726 | if not isRadarFolder(thisPath): | |
| 726 | continue |
|
727 | continue | |
| 727 |
|
728 | |||
| 728 | dirList.append(thisPath) |
|
729 | dirList.append(thisPath) | |
| 729 |
|
730 | |||
| 730 | if not(dirList): |
|
731 | if not(dirList): | |
| 731 | return None, None, None, None, None, None |
|
732 | return None, None, None, None, None, None | |
| 732 |
|
733 | |||
| 733 | dirList = sorted( dirList, key=str.lower ) |
|
734 | dirList = sorted( dirList, key=str.lower ) | |
| 734 |
|
735 | |||
| 735 | doypath = dirList[-1] |
|
736 | doypath = dirList[-1] | |
| 736 | foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0 |
|
737 | foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0 | |
| 737 | fullpath = os.path.join(path, doypath, expLabel) |
|
738 | fullpath = os.path.join(path, doypath, expLabel) | |
| 738 |
|
739 | |||
| 739 |
|
740 | |||
| 740 | print "[Reading] %s folder was found: " %(fullpath ) |
|
741 | print "[Reading] %s folder was found: " %(fullpath ) | |
| 741 |
|
742 | |||
| 742 | if set == None: |
|
743 | if set == None: | |
| 743 | filename = getlastFileFromPath(fullpath, ext) |
|
744 | filename = getlastFileFromPath(fullpath, ext) | |
| 744 | else: |
|
745 | else: | |
| 745 | filename = getFileFromSet(fullpath, ext, set) |
|
746 | filename = getFileFromSet(fullpath, ext, set) | |
| 746 |
|
747 | |||
| 747 | if not(filename): |
|
748 | if not(filename): | |
| 748 | return None, None, None, None, None, None |
|
749 | return None, None, None, None, None, None | |
| 749 |
|
750 | |||
| 750 | print "[Reading] %s file was found" %(filename) |
|
751 | print "[Reading] %s file was found" %(filename) | |
| 751 |
|
752 | |||
| 752 | if not(self.__verifyFile(os.path.join(fullpath, filename))): |
|
753 | if not(self.__verifyFile(os.path.join(fullpath, filename))): | |
| 753 | return None, None, None, None, None, None |
|
754 | return None, None, None, None, None, None | |
| 754 |
|
755 | |||
| 755 | year = int( filename[1:5] ) |
|
756 | year = int( filename[1:5] ) | |
| 756 | doy = int( filename[5:8] ) |
|
757 | doy = int( filename[5:8] ) | |
| 757 | set = int( filename[8:11] ) |
|
758 | set = int( filename[8:11] ) | |
| 758 |
|
759 | |||
| 759 | return fullpath, foldercounter, filename, year, doy, set |
|
760 | return fullpath, foldercounter, filename, year, doy, set | |
| 760 |
|
761 | |||
| 761 | def __setNextFileOffline(self): |
|
762 | def __setNextFileOffline(self): | |
| 762 |
|
763 | |||
| 763 | idFile = self.fileIndex |
|
764 | idFile = self.fileIndex | |
| 764 |
|
765 | |||
| 765 | while (True): |
|
766 | while (True): | |
| 766 | idFile += 1 |
|
767 | idFile += 1 | |
| 767 | if not(idFile < len(self.filenameList)): |
|
768 | if not(idFile < len(self.filenameList)): | |
| 768 | self.flagNoMoreFiles = 1 |
|
769 | self.flagNoMoreFiles = 1 | |
| 769 | # print "[Reading] No more Files" |
|
770 | # print "[Reading] No more Files" | |
| 770 | return 0 |
|
771 | return 0 | |
| 771 |
|
772 | |||
| 772 | filename = self.filenameList[idFile] |
|
773 | filename = self.filenameList[idFile] | |
| 773 |
|
774 | |||
| 774 | if not(self.__verifyFile(filename)): |
|
775 | if not(self.__verifyFile(filename)): | |
| 775 | continue |
|
776 | continue | |
| 776 |
|
777 | |||
| 777 | fileSize = os.path.getsize(filename) |
|
778 | fileSize = os.path.getsize(filename) | |
| 778 | fp = open(filename,'rb') |
|
779 | fp = open(filename,'rb') | |
| 779 | break |
|
780 | break | |
| 780 |
|
781 | |||
| 781 | self.flagIsNewFile = 1 |
|
782 | self.flagIsNewFile = 1 | |
| 782 | self.fileIndex = idFile |
|
783 | self.fileIndex = idFile | |
| 783 | self.filename = filename |
|
784 | self.filename = filename | |
| 784 | self.fileSize = fileSize |
|
785 | self.fileSize = fileSize | |
| 785 | self.fp = fp |
|
786 | self.fp = fp | |
| 786 |
|
787 | |||
| 787 | # print "[Reading] Setting the file: %s"%self.filename |
|
788 | # print "[Reading] Setting the file: %s"%self.filename | |
| 788 |
|
789 | |||
| 789 | return 1 |
|
790 | return 1 | |
| 790 |
|
791 | |||
| 791 | def __setNextFileOnline(self): |
|
792 | def __setNextFileOnline(self): | |
| 792 | """ |
|
793 | """ | |
| 793 | Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si |
|
794 | Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si | |
| 794 | no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files |
|
795 | no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files | |
| 795 | siguientes. |
|
796 | siguientes. | |
| 796 |
|
797 | |||
| 797 | Affected: |
|
798 | Affected: | |
| 798 | self.flagIsNewFile |
|
799 | self.flagIsNewFile | |
| 799 | self.filename |
|
800 | self.filename | |
| 800 | self.fileSize |
|
801 | self.fileSize | |
| 801 | self.fp |
|
802 | self.fp | |
| 802 | self.set |
|
803 | self.set | |
| 803 | self.flagNoMoreFiles |
|
804 | self.flagNoMoreFiles | |
| 804 |
|
805 | |||
| 805 | Return: |
|
806 | Return: | |
| 806 | 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado |
|
807 | 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado | |
| 807 | 1 : si el file fue abierto con exito y esta listo a ser leido |
|
808 | 1 : si el file fue abierto con exito y esta listo a ser leido | |
| 808 |
|
809 | |||
| 809 | Excepciones: |
|
810 | Excepciones: | |
| 810 | Si un determinado file no puede ser abierto |
|
811 | Si un determinado file no puede ser abierto | |
| 811 | """ |
|
812 | """ | |
| 812 | nFiles = 0 |
|
813 | nFiles = 0 | |
| 813 | fileOk_flag = False |
|
814 | fileOk_flag = False | |
| 814 | firstTime_flag = True |
|
815 | firstTime_flag = True | |
| 815 |
|
816 | |||
| 816 | self.set += 1 |
|
817 | self.set += 1 | |
| 817 |
|
818 | |||
| 818 | if self.set > 999: |
|
819 | if self.set > 999: | |
| 819 | self.set = 0 |
|
820 | self.set = 0 | |
| 820 | self.foldercounter += 1 |
|
821 | self.foldercounter += 1 | |
| 821 |
|
822 | |||
| 822 | #busca el 1er file disponible |
|
823 | #busca el 1er file disponible | |
| 823 | fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext ) |
|
824 | fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext ) | |
| 824 | if fullfilename: |
|
825 | if fullfilename: | |
| 825 | if self.__verifyFile(fullfilename, False): |
|
826 | if self.__verifyFile(fullfilename, False): | |
| 826 | fileOk_flag = True |
|
827 | fileOk_flag = True | |
| 827 |
|
828 | |||
| 828 | #si no encuentra un file entonces espera y vuelve a buscar |
|
829 | #si no encuentra un file entonces espera y vuelve a buscar | |
| 829 | if not(fileOk_flag): |
|
830 | if not(fileOk_flag): | |
| 830 | for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles |
|
831 | for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles | |
| 831 |
|
832 | |||
| 832 | if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces |
|
833 | if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces | |
| 833 | tries = self.nTries |
|
834 | tries = self.nTries | |
| 834 | else: |
|
835 | else: | |
| 835 | tries = 1 #si no es la 1era vez entonces solo lo hace una vez |
|
836 | tries = 1 #si no es la 1era vez entonces solo lo hace una vez | |
| 836 |
|
837 | |||
| 837 | for nTries in range( tries ): |
|
838 | for nTries in range( tries ): | |
| 838 | if firstTime_flag: |
|
839 | if firstTime_flag: | |
| 839 | print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 ) |
|
840 | print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 ) | |
| 840 | sleep( self.delay ) |
|
841 | sleep( self.delay ) | |
| 841 | else: |
|
842 | else: | |
| 842 | print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext) |
|
843 | print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext) | |
| 843 |
|
844 | |||
| 844 | fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext ) |
|
845 | fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext ) | |
| 845 | if fullfilename: |
|
846 | if fullfilename: | |
| 846 | if self.__verifyFile(fullfilename): |
|
847 | if self.__verifyFile(fullfilename): | |
| 847 | fileOk_flag = True |
|
848 | fileOk_flag = True | |
| 848 | break |
|
849 | break | |
| 849 |
|
850 | |||
| 850 | if fileOk_flag: |
|
851 | if fileOk_flag: | |
| 851 | break |
|
852 | break | |
| 852 |
|
853 | |||
| 853 | firstTime_flag = False |
|
854 | firstTime_flag = False | |
| 854 |
|
855 | |||
| 855 | print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename |
|
856 | print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename | |
| 856 | self.set += 1 |
|
857 | self.set += 1 | |
| 857 |
|
858 | |||
| 858 | if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta |
|
859 | if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta | |
| 859 | self.set = 0 |
|
860 | self.set = 0 | |
| 860 | self.doy += 1 |
|
861 | self.doy += 1 | |
| 861 | self.foldercounter = 0 |
|
862 | self.foldercounter = 0 | |
| 862 |
|
863 | |||
| 863 | if fileOk_flag: |
|
864 | if fileOk_flag: | |
| 864 | self.fileSize = os.path.getsize( fullfilename ) |
|
865 | self.fileSize = os.path.getsize( fullfilename ) | |
| 865 | self.filename = fullfilename |
|
866 | self.filename = fullfilename | |
| 866 | self.flagIsNewFile = 1 |
|
867 | self.flagIsNewFile = 1 | |
| 867 | if self.fp != None: self.fp.close() |
|
868 | if self.fp != None: self.fp.close() | |
| 868 | self.fp = open(fullfilename, 'rb') |
|
869 | self.fp = open(fullfilename, 'rb') | |
| 869 | self.flagNoMoreFiles = 0 |
|
870 | self.flagNoMoreFiles = 0 | |
| 870 | # print '[Reading] Setting the file: %s' % fullfilename |
|
871 | # print '[Reading] Setting the file: %s' % fullfilename | |
| 871 | else: |
|
872 | else: | |
| 872 | self.fileSize = 0 |
|
873 | self.fileSize = 0 | |
| 873 | self.filename = None |
|
874 | self.filename = None | |
| 874 | self.flagIsNewFile = 0 |
|
875 | self.flagIsNewFile = 0 | |
| 875 | self.fp = None |
|
876 | self.fp = None | |
| 876 | self.flagNoMoreFiles = 1 |
|
877 | self.flagNoMoreFiles = 1 | |
| 877 | # print '[Reading] No more files to read' |
|
878 | # print '[Reading] No more files to read' | |
| 878 |
|
879 | |||
| 879 | return fileOk_flag |
|
880 | return fileOk_flag | |
| 880 |
|
881 | |||
| 881 | def setNextFile(self): |
|
882 | def setNextFile(self): | |
| 882 | if self.fp != None: |
|
883 | if self.fp != None: | |
| 883 | self.fp.close() |
|
884 | self.fp.close() | |
| 884 |
|
885 | |||
| 885 | if self.online: |
|
886 | if self.online: | |
| 886 | newFile = self.__setNextFileOnline() |
|
887 | newFile = self.__setNextFileOnline() | |
| 887 | else: |
|
888 | else: | |
| 888 | newFile = self.__setNextFileOffline() |
|
889 | newFile = self.__setNextFileOffline() | |
| 889 |
|
890 | |||
| 890 | if not(newFile): |
|
891 | if not(newFile): | |
| 891 | print '[Reading] No more files to read' |
|
892 | print '[Reading] No more files to read' | |
| 892 | return 0 |
|
893 | return 0 | |
| 893 |
|
894 | |||
| 894 | if self.verbose: |
|
895 | if self.verbose: | |
| 895 | print '[Reading] Setting the file: %s' % self.filename |
|
896 | print '[Reading] Setting the file: %s' % self.filename | |
| 896 |
|
897 | |||
| 897 | self.__readFirstHeader() |
|
898 | self.__readFirstHeader() | |
| 898 | self.nReadBlocks = 0 |
|
899 | self.nReadBlocks = 0 | |
| 899 | return 1 |
|
900 | return 1 | |
| 900 |
|
901 | |||
| 901 | def __waitNewBlock(self): |
|
902 | def __waitNewBlock(self): | |
| 902 | """ |
|
903 | """ | |
| 903 | Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma. |
|
904 | Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma. | |
| 904 |
|
905 | |||
| 905 | Si el modo de lectura es OffLine siempre retorn 0 |
|
906 | Si el modo de lectura es OffLine siempre retorn 0 | |
| 906 | """ |
|
907 | """ | |
| 907 | if not self.online: |
|
908 | if not self.online: | |
| 908 | return 0 |
|
909 | return 0 | |
| 909 |
|
910 | |||
| 910 | if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile): |
|
911 | if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile): | |
| 911 | return 0 |
|
912 | return 0 | |
| 912 |
|
913 | |||
| 913 | currentPointer = self.fp.tell() |
|
914 | currentPointer = self.fp.tell() | |
| 914 |
|
915 | |||
| 915 | neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize |
|
916 | neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize | |
| 916 |
|
917 | |||
| 917 | for nTries in range( self.nTries ): |
|
918 | for nTries in range( self.nTries ): | |
| 918 |
|
919 | |||
| 919 | self.fp.close() |
|
920 | self.fp.close() | |
| 920 | self.fp = open( self.filename, 'rb' ) |
|
921 | self.fp = open( self.filename, 'rb' ) | |
| 921 | self.fp.seek( currentPointer ) |
|
922 | self.fp.seek( currentPointer ) | |
| 922 |
|
923 | |||
| 923 | self.fileSize = os.path.getsize( self.filename ) |
|
924 | self.fileSize = os.path.getsize( self.filename ) | |
| 924 | currentSize = self.fileSize - currentPointer |
|
925 | currentSize = self.fileSize - currentPointer | |
| 925 |
|
926 | |||
| 926 | if ( currentSize >= neededSize ): |
|
927 | if ( currentSize >= neededSize ): | |
| 927 | self.basicHeaderObj.read(self.fp) |
|
928 | self.basicHeaderObj.read(self.fp) | |
| 928 | return 1 |
|
929 | return 1 | |
| 929 |
|
930 | |||
| 930 | if self.fileSize == self.fileSizeByHeader: |
|
931 | if self.fileSize == self.fileSizeByHeader: | |
| 931 | # self.flagEoF = True |
|
932 | # self.flagEoF = True | |
| 932 | return 0 |
|
933 | return 0 | |
| 933 |
|
934 | |||
| 934 | print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1) |
|
935 | print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1) | |
| 935 | sleep( self.delay ) |
|
936 | sleep( self.delay ) | |
| 936 |
|
937 | |||
| 937 |
|
938 | |||
| 938 | return 0 |
|
939 | return 0 | |
| 939 |
|
940 | |||
| 940 | def waitDataBlock(self,pointer_location): |
|
941 | def waitDataBlock(self,pointer_location): | |
| 941 |
|
942 | |||
| 942 | currentPointer = pointer_location |
|
943 | currentPointer = pointer_location | |
| 943 |
|
944 | |||
| 944 | neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize |
|
945 | neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize | |
| 945 |
|
946 | |||
| 946 | for nTries in range( self.nTries ): |
|
947 | for nTries in range( self.nTries ): | |
| 947 | self.fp.close() |
|
948 | self.fp.close() | |
| 948 | self.fp = open( self.filename, 'rb' ) |
|
949 | self.fp = open( self.filename, 'rb' ) | |
| 949 | self.fp.seek( currentPointer ) |
|
950 | self.fp.seek( currentPointer ) | |
| 950 |
|
951 | |||
| 951 | self.fileSize = os.path.getsize( self.filename ) |
|
952 | self.fileSize = os.path.getsize( self.filename ) | |
| 952 | currentSize = self.fileSize - currentPointer |
|
953 | currentSize = self.fileSize - currentPointer | |
| 953 |
|
954 | |||
| 954 | if ( currentSize >= neededSize ): |
|
955 | if ( currentSize >= neededSize ): | |
| 955 | return 1 |
|
956 | return 1 | |
| 956 |
|
957 | |||
| 957 | print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1) |
|
958 | print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1) | |
| 958 | sleep( self.delay ) |
|
959 | sleep( self.delay ) | |
| 959 |
|
960 | |||
| 960 | return 0 |
|
961 | return 0 | |
| 961 |
|
962 | |||
| 962 | def __jumpToLastBlock(self): |
|
963 | def __jumpToLastBlock(self): | |
| 963 |
|
964 | |||
| 964 | if not(self.__isFirstTimeOnline): |
|
965 | if not(self.__isFirstTimeOnline): | |
| 965 | return |
|
966 | return | |
| 966 |
|
967 | |||
| 967 | csize = self.fileSize - self.fp.tell() |
|
968 | csize = self.fileSize - self.fp.tell() | |
| 968 | blocksize = self.processingHeaderObj.blockSize |
|
969 | blocksize = self.processingHeaderObj.blockSize | |
| 969 |
|
970 | |||
| 970 | #salta el primer bloque de datos |
|
971 | #salta el primer bloque de datos | |
| 971 | if csize > self.processingHeaderObj.blockSize: |
|
972 | if csize > self.processingHeaderObj.blockSize: | |
| 972 | self.fp.seek(self.fp.tell() + blocksize) |
|
973 | self.fp.seek(self.fp.tell() + blocksize) | |
| 973 | else: |
|
974 | else: | |
| 974 | return |
|
975 | return | |
| 975 |
|
976 | |||
| 976 | csize = self.fileSize - self.fp.tell() |
|
977 | csize = self.fileSize - self.fp.tell() | |
| 977 | neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize |
|
978 | neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize | |
| 978 | while True: |
|
979 | while True: | |
| 979 |
|
980 | |||
| 980 | if self.fp.tell()<self.fileSize: |
|
981 | if self.fp.tell()<self.fileSize: | |
| 981 | self.fp.seek(self.fp.tell() + neededsize) |
|
982 | self.fp.seek(self.fp.tell() + neededsize) | |
| 982 | else: |
|
983 | else: | |
| 983 | self.fp.seek(self.fp.tell() - neededsize) |
|
984 | self.fp.seek(self.fp.tell() - neededsize) | |
| 984 | break |
|
985 | break | |
| 985 |
|
986 | |||
| 986 | # csize = self.fileSize - self.fp.tell() |
|
987 | # csize = self.fileSize - self.fp.tell() | |
| 987 | # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize |
|
988 | # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize | |
| 988 | # factor = int(csize/neededsize) |
|
989 | # factor = int(csize/neededsize) | |
| 989 | # if factor > 0: |
|
990 | # if factor > 0: | |
| 990 | # self.fp.seek(self.fp.tell() + factor*neededsize) |
|
991 | # self.fp.seek(self.fp.tell() + factor*neededsize) | |
| 991 |
|
992 | |||
| 992 | self.flagIsNewFile = 0 |
|
993 | self.flagIsNewFile = 0 | |
| 993 | self.__isFirstTimeOnline = 0 |
|
994 | self.__isFirstTimeOnline = 0 | |
| 994 |
|
995 | |||
| 995 | def __setNewBlock(self): |
|
996 | def __setNewBlock(self): | |
| 996 |
|
997 | if self.server is None: | ||
| 997 | if self.fp == None: |
|
998 | if self.fp == None: | |
| 998 | return 0 |
|
999 | return 0 | |
| 999 |
|
1000 | |||
| 1000 | # if self.online: |
|
1001 | # if self.online: | |
| 1001 | # self.__jumpToLastBlock() |
|
1002 | # self.__jumpToLastBlock() | |
|
|
1003 | print 'xxxx' | |||
| 1002 |
|
1004 | |||
| 1003 | if self.flagIsNewFile: |
|
1005 | if self.flagIsNewFile: | |
| 1004 | self.lastUTTime = self.basicHeaderObj.utc |
|
1006 | self.lastUTTime = self.basicHeaderObj.utc | |
| 1005 | return 1 |
|
1007 | return 1 | |
| 1006 |
|
1008 | |||
| 1007 | if self.realtime: |
|
1009 | if self.realtime: | |
| 1008 | self.flagDiscontinuousBlock = 1 |
|
1010 | self.flagDiscontinuousBlock = 1 | |
| 1009 | if not(self.setNextFile()): |
|
1011 | if not(self.setNextFile()): | |
| 1010 | return 0 |
|
1012 | return 0 | |
| 1011 | else: |
|
1013 | else: | |
| 1012 | return 1 |
|
1014 | return 1 | |
| 1013 |
|
1015 | print 'xxxx' | ||
| 1014 | currentSize = self.fileSize - self.fp.tell() |
|
1016 | if self.server is None: | |
| 1015 | neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize |
|
1017 | currentSize = self.fileSize - self.fp.tell() | |
| 1016 |
|
1018 | neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize | ||
| 1017 | if (currentSize >= neededSize): |
|
1019 | if (currentSize >= neededSize): | |
| 1018 | self.basicHeaderObj.read(self.fp) |
|
1020 | self.basicHeaderObj.read(self.fp) | |
|
|
1021 | self.lastUTTime = self.basicHeaderObj.utc | |||
|
|
1022 | return 1 | |||
|
|
1023 | else: | |||
|
|
1024 | self.basicHeaderObj.read(self.zHeader) | |||
| 1019 | self.lastUTTime = self.basicHeaderObj.utc |
|
1025 | self.lastUTTime = self.basicHeaderObj.utc | |
| 1020 | return 1 |
|
1026 | return 1 | |
| 1021 |
|
||||
| 1022 | if self.__waitNewBlock(): |
|
1027 | if self.__waitNewBlock(): | |
| 1023 | self.lastUTTime = self.basicHeaderObj.utc |
|
1028 | self.lastUTTime = self.basicHeaderObj.utc | |
| 1024 | return 1 |
|
1029 | return 1 | |
| 1025 |
|
1030 | if self.server is None: | ||
| 1026 | if not(self.setNextFile()): |
|
1031 | if not(self.setNextFile()): | |
| 1027 | return 0 |
|
1032 | return 0 | |
| 1028 |
|
1033 | |||
| 1029 | deltaTime = self.basicHeaderObj.utc - self.lastUTTime # |
|
1034 | deltaTime = self.basicHeaderObj.utc - self.lastUTTime # | |
| 1030 | self.lastUTTime = self.basicHeaderObj.utc |
|
1035 | self.lastUTTime = self.basicHeaderObj.utc | |
| 1031 |
|
1036 | |||
| 1032 | self.flagDiscontinuousBlock = 0 |
|
1037 | self.flagDiscontinuousBlock = 0 | |
| 1033 |
|
1038 | |||
| 1034 | if deltaTime > self.maxTimeStep: |
|
1039 | if deltaTime > self.maxTimeStep: | |
| 1035 | self.flagDiscontinuousBlock = 1 |
|
1040 | self.flagDiscontinuousBlock = 1 | |
| 1036 |
|
1041 | |||
| 1037 | return 1 |
|
1042 | return 1 | |
| 1038 |
|
1043 | |||
| 1039 | def readNextBlock(self): |
|
1044 | def readNextBlock(self): | |
| 1040 |
|
1045 | |||
| 1041 | #Skip block out of startTime and endTime |
|
1046 | #Skip block out of startTime and endTime | |
| 1042 | while True: |
|
1047 | while True: | |
|
|
1048 | print 'cxxxx' | |||
| 1043 | if not(self.__setNewBlock()): |
|
1049 | if not(self.__setNewBlock()): | |
|
|
1050 | print 'returning' | |||
| 1044 | return 0 |
|
1051 | return 0 | |
| 1045 |
|
1052 | print 'dxxx' | ||
| 1046 | if not(self.readBlock()): |
|
1053 | if not(self.readBlock()): | |
| 1047 | return 0 |
|
1054 | return 0 | |
| 1048 |
|
1055 | |||
| 1049 | self.getBasicHeader() |
|
1056 | self.getBasicHeader() | |
| 1050 |
|
1057 | |||
| 1051 | if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime): |
|
1058 | if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime): | |
| 1052 |
|
1059 | |||
| 1053 | print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks, |
|
1060 | print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks, | |
| 1054 | self.processingHeaderObj.dataBlocksPerFile, |
|
1061 | self.processingHeaderObj.dataBlocksPerFile, | |
| 1055 | self.dataOut.datatime.ctime()) |
|
1062 | self.dataOut.datatime.ctime()) | |
| 1056 | continue |
|
1063 | continue | |
| 1057 |
|
1064 | |||
| 1058 | break |
|
1065 | break | |
| 1059 |
|
1066 | |||
| 1060 | if self.verbose: |
|
1067 | if self.verbose: | |
| 1061 | print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks, |
|
1068 | print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks, | |
| 1062 | self.processingHeaderObj.dataBlocksPerFile, |
|
1069 | self.processingHeaderObj.dataBlocksPerFile, | |
| 1063 | self.dataOut.datatime.ctime()) |
|
1070 | self.dataOut.datatime.ctime()) | |
| 1064 | return 1 |
|
1071 | return 1 | |
| 1065 |
|
1072 | |||
| 1066 | def __readFirstHeader(self): |
|
1073 | def __readFirstHeader(self): | |
| 1067 |
|
1074 | |||
| 1068 | self.basicHeaderObj.read(self.fp) |
|
1075 | self.basicHeaderObj.read(self.fp) | |
| 1069 | self.systemHeaderObj.read(self.fp) |
|
1076 | self.systemHeaderObj.read(self.fp) | |
| 1070 | self.radarControllerHeaderObj.read(self.fp) |
|
1077 | self.radarControllerHeaderObj.read(self.fp) | |
| 1071 | self.processingHeaderObj.read(self.fp) |
|
1078 | self.processingHeaderObj.read(self.fp) | |
| 1072 |
|
1079 | |||
| 1073 | self.firstHeaderSize = self.basicHeaderObj.size |
|
1080 | self.firstHeaderSize = self.basicHeaderObj.size | |
| 1074 |
|
1081 | |||
| 1075 | datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR)) |
|
1082 | datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR)) | |
| 1076 | if datatype == 0: |
|
1083 | if datatype == 0: | |
| 1077 | datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')]) |
|
1084 | datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')]) | |
| 1078 | elif datatype == 1: |
|
1085 | elif datatype == 1: | |
| 1079 | datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')]) |
|
1086 | datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')]) | |
| 1080 | elif datatype == 2: |
|
1087 | elif datatype == 2: | |
| 1081 | datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')]) |
|
1088 | datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')]) | |
| 1082 | elif datatype == 3: |
|
1089 | elif datatype == 3: | |
| 1083 | datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')]) |
|
1090 | datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')]) | |
| 1084 | elif datatype == 4: |
|
1091 | elif datatype == 4: | |
| 1085 | datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')]) |
|
1092 | datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')]) | |
| 1086 | elif datatype == 5: |
|
1093 | elif datatype == 5: | |
| 1087 | datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')]) |
|
1094 | datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')]) | |
| 1088 | else: |
|
1095 | else: | |
| 1089 | raise ValueError, 'Data type was not defined' |
|
1096 | raise ValueError, 'Data type was not defined' | |
| 1090 |
|
1097 | |||
| 1091 | self.dtype = datatype_str |
|
1098 | self.dtype = datatype_str | |
| 1092 | #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c |
|
1099 | #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c | |
| 1093 | self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1) |
|
1100 | self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1) | |
| 1094 | # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels) |
|
1101 | # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels) | |
| 1095 | # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels) |
|
1102 | # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels) | |
| 1096 | self.getBlockDimension() |
|
1103 | self.getBlockDimension() | |
| 1097 |
|
1104 | |||
| 1098 | def __verifyFile(self, filename, msgFlag=True): |
|
1105 | def __verifyFile(self, filename, msgFlag=True): | |
| 1099 |
|
1106 | |||
| 1100 | msg = None |
|
1107 | msg = None | |
| 1101 |
|
1108 | |||
| 1102 | try: |
|
1109 | try: | |
| 1103 | fp = open(filename, 'rb') |
|
1110 | fp = open(filename, 'rb') | |
| 1104 | except IOError: |
|
1111 | except IOError: | |
| 1105 |
|
1112 | |||
| 1106 | if msgFlag: |
|
1113 | if msgFlag: | |
| 1107 | print "[Reading] File %s can't be opened" % (filename) |
|
1114 | print "[Reading] File %s can't be opened" % (filename) | |
| 1108 |
|
1115 | |||
| 1109 | return False |
|
1116 | return False | |
| 1110 |
|
1117 | |||
| 1111 | currentPosition = fp.tell() |
|
1118 | currentPosition = fp.tell() | |
| 1112 | neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize |
|
1119 | neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize | |
| 1113 |
|
1120 | |||
| 1114 | if neededSize == 0: |
|
1121 | if neededSize == 0: | |
| 1115 | basicHeaderObj = BasicHeader(LOCALTIME) |
|
1122 | basicHeaderObj = BasicHeader(LOCALTIME) | |
| 1116 | systemHeaderObj = SystemHeader() |
|
1123 | systemHeaderObj = SystemHeader() | |
| 1117 | radarControllerHeaderObj = RadarControllerHeader() |
|
1124 | radarControllerHeaderObj = RadarControllerHeader() | |
| 1118 | processingHeaderObj = ProcessingHeader() |
|
1125 | processingHeaderObj = ProcessingHeader() | |
| 1119 |
|
1126 | |||
| 1120 | if not( basicHeaderObj.read(fp) ): |
|
1127 | if not( basicHeaderObj.read(fp) ): | |
| 1121 | fp.close() |
|
1128 | fp.close() | |
| 1122 | return False |
|
1129 | return False | |
| 1123 |
|
1130 | |||
| 1124 | if not( systemHeaderObj.read(fp) ): |
|
1131 | if not( systemHeaderObj.read(fp) ): | |
| 1125 | fp.close() |
|
1132 | fp.close() | |
| 1126 | return False |
|
1133 | return False | |
| 1127 |
|
1134 | |||
| 1128 | if not( radarControllerHeaderObj.read(fp) ): |
|
1135 | if not( radarControllerHeaderObj.read(fp) ): | |
| 1129 | fp.close() |
|
1136 | fp.close() | |
| 1130 | return False |
|
1137 | return False | |
| 1131 |
|
1138 | |||
| 1132 | if not( processingHeaderObj.read(fp) ): |
|
1139 | if not( processingHeaderObj.read(fp) ): | |
| 1133 | fp.close() |
|
1140 | fp.close() | |
| 1134 | return False |
|
1141 | return False | |
| 1135 |
|
1142 | |||
| 1136 | neededSize = processingHeaderObj.blockSize + basicHeaderObj.size |
|
1143 | neededSize = processingHeaderObj.blockSize + basicHeaderObj.size | |
| 1137 | else: |
|
1144 | else: | |
| 1138 | msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename |
|
1145 | msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename | |
| 1139 |
|
1146 | |||
| 1140 | fp.close() |
|
1147 | fp.close() | |
| 1141 |
|
1148 | |||
| 1142 | fileSize = os.path.getsize(filename) |
|
1149 | fileSize = os.path.getsize(filename) | |
| 1143 | currentSize = fileSize - currentPosition |
|
1150 | currentSize = fileSize - currentPosition | |
| 1144 |
|
1151 | |||
| 1145 | if currentSize < neededSize: |
|
1152 | if currentSize < neededSize: | |
| 1146 | if msgFlag and (msg != None): |
|
1153 | if msgFlag and (msg != None): | |
| 1147 | print msg |
|
1154 | print msg | |
| 1148 | return False |
|
1155 | return False | |
| 1149 |
|
1156 | |||
| 1150 | return True |
|
1157 | return True | |
| 1151 |
|
1158 | |||
| 1152 | def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False): |
|
1159 | def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False): | |
| 1153 |
|
1160 | |||
| 1154 | path_empty = True |
|
1161 | path_empty = True | |
| 1155 |
|
1162 | |||
| 1156 | dateList = [] |
|
1163 | dateList = [] | |
| 1157 | pathList = [] |
|
1164 | pathList = [] | |
| 1158 |
|
1165 | |||
| 1159 | multi_path = path.split(',') |
|
1166 | multi_path = path.split(',') | |
| 1160 |
|
1167 | |||
| 1161 | if not walk: |
|
1168 | if not walk: | |
| 1162 |
|
1169 | |||
| 1163 | for single_path in multi_path: |
|
1170 | for single_path in multi_path: | |
| 1164 |
|
1171 | |||
| 1165 | if not os.path.isdir(single_path): |
|
1172 | if not os.path.isdir(single_path): | |
| 1166 | continue |
|
1173 | continue | |
| 1167 |
|
1174 | |||
| 1168 | fileList = glob.glob1(single_path, "*"+ext) |
|
1175 | fileList = glob.glob1(single_path, "*"+ext) | |
| 1169 |
|
1176 | |||
| 1170 | if not fileList: |
|
1177 | if not fileList: | |
| 1171 | continue |
|
1178 | continue | |
| 1172 |
|
1179 | |||
| 1173 | path_empty = False |
|
1180 | path_empty = False | |
| 1174 |
|
1181 | |||
| 1175 | fileList.sort() |
|
1182 | fileList.sort() | |
| 1176 |
|
1183 | |||
| 1177 | for thisFile in fileList: |
|
1184 | for thisFile in fileList: | |
| 1178 |
|
1185 | |||
| 1179 | if not os.path.isfile(os.path.join(single_path, thisFile)): |
|
1186 | if not os.path.isfile(os.path.join(single_path, thisFile)): | |
| 1180 | continue |
|
1187 | continue | |
| 1181 |
|
1188 | |||
| 1182 | if not isRadarFile(thisFile): |
|
1189 | if not isRadarFile(thisFile): | |
| 1183 | continue |
|
1190 | continue | |
| 1184 |
|
1191 | |||
| 1185 | if not isFileInDateRange(thisFile, startDate, endDate): |
|
1192 | if not isFileInDateRange(thisFile, startDate, endDate): | |
| 1186 | continue |
|
1193 | continue | |
| 1187 |
|
1194 | |||
| 1188 | thisDate = getDateFromRadarFile(thisFile) |
|
1195 | thisDate = getDateFromRadarFile(thisFile) | |
| 1189 |
|
1196 | |||
| 1190 | if thisDate in dateList: |
|
1197 | if thisDate in dateList: | |
| 1191 | continue |
|
1198 | continue | |
| 1192 |
|
1199 | |||
| 1193 | dateList.append(thisDate) |
|
1200 | dateList.append(thisDate) | |
| 1194 | pathList.append(single_path) |
|
1201 | pathList.append(single_path) | |
| 1195 |
|
1202 | |||
| 1196 | else: |
|
1203 | else: | |
| 1197 | for single_path in multi_path: |
|
1204 | for single_path in multi_path: | |
| 1198 |
|
1205 | |||
| 1199 | if not os.path.isdir(single_path): |
|
1206 | if not os.path.isdir(single_path): | |
| 1200 | continue |
|
1207 | continue | |
| 1201 |
|
1208 | |||
| 1202 | dirList = [] |
|
1209 | dirList = [] | |
| 1203 |
|
1210 | |||
| 1204 | for thisPath in os.listdir(single_path): |
|
1211 | for thisPath in os.listdir(single_path): | |
| 1205 |
|
1212 | |||
| 1206 | if not os.path.isdir(os.path.join(single_path,thisPath)): |
|
1213 | if not os.path.isdir(os.path.join(single_path,thisPath)): | |
| 1207 | continue |
|
1214 | continue | |
| 1208 |
|
1215 | |||
| 1209 | if not isRadarFolder(thisPath): |
|
1216 | if not isRadarFolder(thisPath): | |
| 1210 | continue |
|
1217 | continue | |
| 1211 |
|
1218 | |||
| 1212 | if not isFolderInDateRange(thisPath, startDate, endDate): |
|
1219 | if not isFolderInDateRange(thisPath, startDate, endDate): | |
| 1213 | continue |
|
1220 | continue | |
| 1214 |
|
1221 | |||
| 1215 | dirList.append(thisPath) |
|
1222 | dirList.append(thisPath) | |
| 1216 |
|
1223 | |||
| 1217 | if not dirList: |
|
1224 | if not dirList: | |
| 1218 | continue |
|
1225 | continue | |
| 1219 |
|
1226 | |||
| 1220 | dirList.sort() |
|
1227 | dirList.sort() | |
| 1221 |
|
1228 | |||
| 1222 | for thisDir in dirList: |
|
1229 | for thisDir in dirList: | |
| 1223 |
|
1230 | |||
| 1224 | datapath = os.path.join(single_path, thisDir, expLabel) |
|
1231 | datapath = os.path.join(single_path, thisDir, expLabel) | |
| 1225 | fileList = glob.glob1(datapath, "*"+ext) |
|
1232 | fileList = glob.glob1(datapath, "*"+ext) | |
| 1226 |
|
1233 | |||
| 1227 | if not fileList: |
|
1234 | if not fileList: | |
| 1228 | continue |
|
1235 | continue | |
| 1229 |
|
1236 | |||
| 1230 | path_empty = False |
|
1237 | path_empty = False | |
| 1231 |
|
1238 | |||
| 1232 | thisDate = getDateFromRadarFolder(thisDir) |
|
1239 | thisDate = getDateFromRadarFolder(thisDir) | |
| 1233 |
|
1240 | |||
| 1234 | pathList.append(datapath) |
|
1241 | pathList.append(datapath) | |
| 1235 | dateList.append(thisDate) |
|
1242 | dateList.append(thisDate) | |
| 1236 |
|
1243 | |||
| 1237 | dateList.sort() |
|
1244 | dateList.sort() | |
| 1238 |
|
1245 | |||
| 1239 | if walk: |
|
1246 | if walk: | |
| 1240 | pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel) |
|
1247 | pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel) | |
| 1241 | else: |
|
1248 | else: | |
| 1242 | pattern_path = multi_path[0] |
|
1249 | pattern_path = multi_path[0] | |
| 1243 |
|
1250 | |||
| 1244 | if path_empty: |
|
1251 | if path_empty: | |
| 1245 | print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate) |
|
1252 | print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate) | |
| 1246 | else: |
|
1253 | else: | |
| 1247 | if not dateList: |
|
1254 | if not dateList: | |
| 1248 | print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path) |
|
1255 | print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path) | |
| 1249 |
|
1256 | |||
| 1250 | if include_path: |
|
1257 | if include_path: | |
| 1251 | return dateList, pathList |
|
1258 | return dateList, pathList | |
| 1252 |
|
1259 | |||
| 1253 | return dateList |
|
1260 | return dateList | |
| 1254 |
|
1261 | |||
| 1255 | def setup(self, |
|
1262 | def setup(self, | |
| 1256 | path=None, |
|
1263 | path=None, | |
| 1257 | startDate=None, |
|
1264 | startDate=None, | |
| 1258 | endDate=None, |
|
1265 | endDate=None, | |
| 1259 | startTime=datetime.time(0,0,0), |
|
1266 | startTime=datetime.time(0,0,0), | |
| 1260 | endTime=datetime.time(23,59,59), |
|
1267 | endTime=datetime.time(23,59,59), | |
| 1261 | set=None, |
|
1268 | set=None, | |
| 1262 | expLabel = "", |
|
1269 | expLabel = "", | |
| 1263 | ext = None, |
|
1270 | ext = None, | |
| 1264 | online = False, |
|
1271 | online = False, | |
| 1265 | delay = 60, |
|
1272 | delay = 60, | |
| 1266 | walk = True, |
|
1273 | walk = True, | |
| 1267 | getblock = False, |
|
1274 | getblock = False, | |
| 1268 | nTxs = 1, |
|
1275 | nTxs = 1, | |
| 1269 | realtime=False, |
|
1276 | realtime=False, | |
| 1270 | blocksize=None, |
|
1277 | blocksize=None, | |
| 1271 | blocktime=None, |
|
1278 | blocktime=None, | |
| 1272 | queue=None, |
|
1279 | queue=None, | |
| 1273 | skip=None, |
|
1280 | skip=None, | |
| 1274 | cursor=None, |
|
1281 | cursor=None, | |
| 1275 | warnings=True, |
|
1282 | warnings=True, | |
| 1276 | verbose=True, |
|
1283 | verbose=True, | |
| 1277 | server=None): |
|
1284 | server=None): | |
| 1278 |
|
||||
| 1279 | if server is not None: |
|
1285 | if server is not None: | |
| 1280 | server = kwargs.get('server', 'zmq.pipe') |
|
|||
| 1281 | if 'tcp://' in server: |
|
1286 | if 'tcp://' in server: | |
| 1282 | address = server |
|
1287 | address = server | |
| 1283 | else: |
|
1288 | else: | |
| 1284 | address = 'ipc:///tmp/%s' % server |
|
1289 | address = 'ipc:///tmp/%s' % server | |
| 1285 |
self. |
|
1290 | self.server = address | |
| 1286 | self.context = zmq.Context() |
|
1291 | self.context = zmq.Context() | |
| 1287 | self.receiver = self.context.socket(zmq.PULL) |
|
1292 | self.receiver = self.context.socket(zmq.PULL) | |
| 1288 |
self.receiver.bind(self. |
|
1293 | self.receiver.bind(self.server) | |
| 1289 | time.sleep(0.5) |
|
1294 | time.sleep(0.5) | |
| 1290 |
print '[Starting] ReceiverData from {}'.format(self. |
|
1295 | print '[Starting] ReceiverData from {}'.format(self.server) | |
| 1291 |
else: |
|
1296 | else: | |
|
|
1297 | self.server = None | |||
| 1292 | if path == None: |
|
1298 | if path == None: | |
| 1293 | raise ValueError, "[Reading] The path is not valid" |
|
1299 | raise ValueError, "[Reading] The path is not valid" | |
| 1294 |
|
1300 | |||
| 1295 | if ext == None: |
|
1301 | if ext == None: | |
| 1296 | ext = self.ext |
|
1302 | ext = self.ext | |
| 1297 |
|
1303 | |||
| 1298 | if online: |
|
1304 | if online: | |
| 1299 | print "[Reading] Searching files in online mode..." |
|
1305 | print "[Reading] Searching files in online mode..." | |
| 1300 |
|
1306 | |||
| 1301 | for nTries in range( self.nTries ): |
|
1307 | for nTries in range( self.nTries ): | |
| 1302 | fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set) |
|
1308 | fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set) | |
| 1303 |
|
1309 | |||
| 1304 | if fullpath: |
|
1310 | if fullpath: | |
| 1305 | break |
|
1311 | break | |
| 1306 |
|
1312 | |||
| 1307 | print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1) |
|
1313 | print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1) | |
| 1308 | sleep( self.delay ) |
|
1314 | sleep( self.delay ) | |
| 1309 |
|
1315 | |||
| 1310 | if not(fullpath): |
|
1316 | if not(fullpath): | |
| 1311 | print "[Reading] There 'isn't any valid file in %s" % path |
|
1317 | print "[Reading] There 'isn't any valid file in %s" % path | |
| 1312 | return |
|
1318 | return | |
| 1313 |
|
1319 | |||
| 1314 | self.year = year |
|
1320 | self.year = year | |
| 1315 | self.doy = doy |
|
1321 | self.doy = doy | |
| 1316 | self.set = set - 1 |
|
1322 | self.set = set - 1 | |
| 1317 | self.path = path |
|
1323 | self.path = path | |
| 1318 | self.foldercounter = foldercounter |
|
1324 | self.foldercounter = foldercounter | |
| 1319 | last_set = None |
|
1325 | last_set = None | |
| 1320 | else: |
|
1326 | else: | |
| 1321 | print "[Reading] Searching files in offline mode ..." |
|
1327 | print "[Reading] Searching files in offline mode ..." | |
| 1322 | pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate, |
|
1328 | pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate, | |
| 1323 | startTime=startTime, endTime=endTime, |
|
1329 | startTime=startTime, endTime=endTime, | |
| 1324 | set=set, expLabel=expLabel, ext=ext, |
|
1330 | set=set, expLabel=expLabel, ext=ext, | |
| 1325 | walk=walk, cursor=cursor, |
|
1331 | walk=walk, cursor=cursor, | |
| 1326 | skip=skip, queue=queue) |
|
1332 | skip=skip, queue=queue) | |
| 1327 |
|
1333 | |||
| 1328 | if not(pathList): |
|
1334 | if not(pathList): | |
| 1329 | # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path, |
|
1335 | # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path, | |
| 1330 | # datetime.datetime.combine(startDate,startTime).ctime(), |
|
1336 | # datetime.datetime.combine(startDate,startTime).ctime(), | |
| 1331 | # datetime.datetime.combine(endDate,endTime).ctime()) |
|
1337 | # datetime.datetime.combine(endDate,endTime).ctime()) | |
| 1332 |
|
1338 | |||
| 1333 | # sys.exit(-1) |
|
1339 | # sys.exit(-1) | |
| 1334 |
|
1340 | |||
| 1335 | self.fileIndex = -1 |
|
1341 | self.fileIndex = -1 | |
| 1336 | self.pathList = [] |
|
1342 | self.pathList = [] | |
| 1337 | self.filenameList = [] |
|
1343 | self.filenameList = [] | |
| 1338 | return |
|
1344 | return | |
| 1339 |
|
1345 | |||
| 1340 | self.fileIndex = -1 |
|
1346 | self.fileIndex = -1 | |
| 1341 | self.pathList = pathList |
|
1347 | self.pathList = pathList | |
| 1342 | self.filenameList = filenameList |
|
1348 | self.filenameList = filenameList | |
| 1343 | file_name = os.path.basename(filenameList[-1]) |
|
1349 | file_name = os.path.basename(filenameList[-1]) | |
| 1344 | basename, ext = os.path.splitext(file_name) |
|
1350 | basename, ext = os.path.splitext(file_name) | |
| 1345 | last_set = int(basename[-3:]) |
|
1351 | last_set = int(basename[-3:]) | |
| 1346 |
|
1352 | |||
| 1347 | self.online = online |
|
1353 | self.online = online | |
| 1348 | self.realtime = realtime |
|
1354 | self.realtime = realtime | |
| 1349 | self.delay = delay |
|
1355 | self.delay = delay | |
| 1350 | ext = ext.lower() |
|
1356 | ext = ext.lower() | |
| 1351 | self.ext = ext |
|
1357 | self.ext = ext | |
| 1352 | self.getByBlock = getblock |
|
1358 | self.getByBlock = getblock | |
| 1353 | self.nTxs = nTxs |
|
1359 | self.nTxs = nTxs | |
| 1354 | self.startTime = startTime |
|
1360 | self.startTime = startTime | |
| 1355 | self.endTime = endTime |
|
1361 | self.endTime = endTime | |
| 1356 |
|
1362 | |||
| 1357 | #Added----------------- |
|
1363 | #Added----------------- | |
| 1358 | self.selBlocksize = blocksize |
|
1364 | self.selBlocksize = blocksize | |
| 1359 | self.selBlocktime = blocktime |
|
1365 | self.selBlocktime = blocktime | |
| 1360 |
|
1366 | |||
| 1361 | # Verbose----------- |
|
1367 | # Verbose----------- | |
| 1362 | self.verbose = verbose |
|
1368 | self.verbose = verbose | |
| 1363 | self.warnings = warnings |
|
1369 | self.warnings = warnings | |
| 1364 |
|
1370 | |||
| 1365 | if not(self.setNextFile()): |
|
1371 | if not(self.setNextFile()): | |
| 1366 | if (startDate!=None) and (endDate!=None): |
|
1372 | if (startDate!=None) and (endDate!=None): | |
| 1367 | print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()) |
|
1373 | print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()) | |
| 1368 | elif startDate != None: |
|
1374 | elif startDate != None: | |
| 1369 | print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime()) |
|
1375 | print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime()) | |
| 1370 | else: |
|
1376 | else: | |
| 1371 | print "[Reading] No files" |
|
1377 | print "[Reading] No files" | |
| 1372 |
|
1378 | |||
| 1373 | self.fileIndex = -1 |
|
1379 | self.fileIndex = -1 | |
| 1374 | self.pathList = [] |
|
1380 | self.pathList = [] | |
| 1375 | self.filenameList = [] |
|
1381 | self.filenameList = [] | |
| 1376 | return |
|
1382 | return | |
| 1377 |
|
1383 | |||
| 1378 | # self.getBasicHeader() |
|
1384 | # self.getBasicHeader() | |
| 1379 |
|
1385 | |||
| 1380 | if last_set != None: |
|
1386 | if last_set != None: | |
| 1381 | self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock |
|
1387 | self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock | |
| 1382 | return |
|
1388 | return | |
| 1383 |
|
1389 | |||
| 1384 | def getBasicHeader(self): |
|
1390 | def getBasicHeader(self): | |
| 1385 |
|
1391 | |||
| 1386 | self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds |
|
1392 | self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds | |
| 1387 |
|
1393 | |||
| 1388 | self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock |
|
1394 | self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock | |
| 1389 |
|
1395 | |||
| 1390 | self.dataOut.timeZone = self.basicHeaderObj.timeZone |
|
1396 | self.dataOut.timeZone = self.basicHeaderObj.timeZone | |
| 1391 |
|
1397 | |||
| 1392 | self.dataOut.dstFlag = self.basicHeaderObj.dstFlag |
|
1398 | self.dataOut.dstFlag = self.basicHeaderObj.dstFlag | |
| 1393 |
|
1399 | |||
| 1394 | self.dataOut.errorCount = self.basicHeaderObj.errorCount |
|
1400 | self.dataOut.errorCount = self.basicHeaderObj.errorCount | |
| 1395 |
|
1401 | |||
| 1396 | self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime |
|
1402 | self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime | |
| 1397 |
|
1403 | |||
| 1398 | self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs |
|
1404 | self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs | |
| 1399 |
|
1405 | |||
| 1400 | # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs |
|
1406 | # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs | |
| 1401 |
|
1407 | |||
| 1402 |
|
1408 | |||
| 1403 | def getFirstHeader(self): |
|
1409 | def getFirstHeader(self): | |
| 1404 |
|
1410 | |||
| 1405 | raise NotImplementedError |
|
1411 | raise NotImplementedError | |
| 1406 |
|
1412 | |||
| 1407 | def getData(self): |
|
1413 | def getData(self): | |
| 1408 |
|
1414 | |||
| 1409 | raise NotImplementedError |
|
1415 | raise NotImplementedError | |
| 1410 |
|
1416 | |||
| 1411 | def hasNotDataInBuffer(self): |
|
1417 | def hasNotDataInBuffer(self): | |
| 1412 |
|
1418 | |||
| 1413 | raise NotImplementedError |
|
1419 | raise NotImplementedError | |
| 1414 |
|
1420 | |||
| 1415 | def readBlock(self): |
|
1421 | def readBlock(self): | |
| 1416 |
|
1422 | |||
| 1417 | raise NotImplementedError |
|
1423 | raise NotImplementedError | |
| 1418 |
|
1424 | |||
| 1419 | def isEndProcess(self): |
|
1425 | def isEndProcess(self): | |
| 1420 |
|
1426 | |||
| 1421 | return self.flagNoMoreFiles |
|
1427 | return self.flagNoMoreFiles | |
| 1422 |
|
1428 | |||
| 1423 | def printReadBlocks(self): |
|
1429 | def printReadBlocks(self): | |
| 1424 |
|
1430 | |||
| 1425 | print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks |
|
1431 | print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks | |
| 1426 |
|
1432 | |||
| 1427 | def printTotalBlocks(self): |
|
1433 | def printTotalBlocks(self): | |
| 1428 |
|
1434 | |||
| 1429 | print "[Reading] Number of read blocks %04d" %self.nTotalBlocks |
|
1435 | print "[Reading] Number of read blocks %04d" %self.nTotalBlocks | |
| 1430 |
|
1436 | |||
| 1431 | def printNumberOfBlock(self): |
|
1437 | def printNumberOfBlock(self): | |
| 1432 |
|
1438 | |||
| 1433 | if self.flagIsNewBlock: |
|
1439 | if self.flagIsNewBlock: | |
| 1434 | print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks, |
|
1440 | print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks, | |
| 1435 | self.processingHeaderObj.dataBlocksPerFile, |
|
1441 | self.processingHeaderObj.dataBlocksPerFile, | |
| 1436 | self.dataOut.datatime.ctime()) |
|
1442 | self.dataOut.datatime.ctime()) | |
| 1437 |
|
1443 | |||
| 1438 | def printInfo(self): |
|
1444 | def printInfo(self): | |
| 1439 |
|
1445 | |||
| 1440 | if self.__printInfo == False: |
|
1446 | if self.__printInfo == False: | |
| 1441 | return |
|
1447 | return | |
| 1442 |
|
1448 | |||
| 1443 | self.basicHeaderObj.printInfo() |
|
1449 | self.basicHeaderObj.printInfo() | |
| 1444 | self.systemHeaderObj.printInfo() |
|
1450 | self.systemHeaderObj.printInfo() | |
| 1445 | self.radarControllerHeaderObj.printInfo() |
|
1451 | self.radarControllerHeaderObj.printInfo() | |
| 1446 | self.processingHeaderObj.printInfo() |
|
1452 | self.processingHeaderObj.printInfo() | |
| 1447 |
|
1453 | |||
| 1448 | self.__printInfo = False |
|
1454 | self.__printInfo = False | |
| 1449 |
|
1455 | |||
| 1450 |
|
1456 | |||
| 1451 | def run(self, |
|
1457 | def run(self, | |
| 1452 | path=None, |
|
1458 | path=None, | |
| 1453 | startDate=None, |
|
1459 | startDate=None, | |
| 1454 | endDate=None, |
|
1460 | endDate=None, | |
| 1455 | startTime=datetime.time(0,0,0), |
|
1461 | startTime=datetime.time(0,0,0), | |
| 1456 | endTime=datetime.time(23,59,59), |
|
1462 | endTime=datetime.time(23,59,59), | |
| 1457 | set=None, |
|
1463 | set=None, | |
| 1458 | expLabel = "", |
|
1464 | expLabel = "", | |
| 1459 | ext = None, |
|
1465 | ext = None, | |
| 1460 | online = False, |
|
1466 | online = False, | |
| 1461 | delay = 60, |
|
1467 | delay = 60, | |
| 1462 | walk = True, |
|
1468 | walk = True, | |
| 1463 | getblock = False, |
|
1469 | getblock = False, | |
| 1464 | nTxs = 1, |
|
1470 | nTxs = 1, | |
| 1465 | realtime=False, |
|
1471 | realtime=False, | |
| 1466 | blocksize=None, |
|
1472 | blocksize=None, | |
| 1467 | blocktime=None, |
|
1473 | blocktime=None, | |
| 1468 | queue=None, |
|
1474 | queue=None, | |
| 1469 | skip=None, |
|
1475 | skip=None, | |
| 1470 | cursor=None, |
|
1476 | cursor=None, | |
| 1471 | warnings=True, |
|
1477 | warnings=True, | |
| 1472 | server=None, |
|
1478 | server=None, | |
| 1473 | verbose=True, **kwargs): |
|
1479 | verbose=True, **kwargs): | |
| 1474 |
|
1480 | |||
| 1475 | if not(self.isConfig): |
|
1481 | if not(self.isConfig): | |
| 1476 | # self.dataOut = dataOut |
|
1482 | # self.dataOut = dataOut | |
| 1477 | self.setup( path=path, |
|
1483 | self.setup( path=path, | |
| 1478 | startDate=startDate, |
|
1484 | startDate=startDate, | |
| 1479 | endDate=endDate, |
|
1485 | endDate=endDate, | |
| 1480 | startTime=startTime, |
|
1486 | startTime=startTime, | |
| 1481 | endTime=endTime, |
|
1487 | endTime=endTime, | |
| 1482 | set=set, |
|
1488 | set=set, | |
| 1483 | expLabel=expLabel, |
|
1489 | expLabel=expLabel, | |
| 1484 | ext=ext, |
|
1490 | ext=ext, | |
| 1485 | online=online, |
|
1491 | online=online, | |
| 1486 | delay=delay, |
|
1492 | delay=delay, | |
| 1487 | walk=walk, |
|
1493 | walk=walk, | |
| 1488 | getblock=getblock, |
|
1494 | getblock=getblock, | |
| 1489 | nTxs=nTxs, |
|
1495 | nTxs=nTxs, | |
| 1490 | realtime=realtime, |
|
1496 | realtime=realtime, | |
| 1491 | blocksize=blocksize, |
|
1497 | blocksize=blocksize, | |
| 1492 | blocktime=blocktime, |
|
1498 | blocktime=blocktime, | |
| 1493 | queue=queue, |
|
1499 | queue=queue, | |
| 1494 | skip=skip, |
|
1500 | skip=skip, | |
| 1495 | cursor=cursor, |
|
1501 | cursor=cursor, | |
| 1496 | warnings=warnings, |
|
1502 | warnings=warnings, | |
| 1497 |
server= |
|
1503 | server=server, | |
| 1498 | verbose=verbose) |
|
1504 | verbose=verbose) | |
| 1499 | self.isConfig = True |
|
1505 | self.isConfig = True | |
| 1500 | if self.server is None: |
|
1506 | print 'hola' | |
| 1501 |
|
|
1507 | self.getData() | |
| 1502 | else: |
|
|||
| 1503 | self.getFromZMQ() |
|
|||
| 1504 |
|
1508 | |||
| 1505 | class JRODataWriter(JRODataIO): |
|
1509 | class JRODataWriter(JRODataIO): | |
| 1506 |
|
1510 | |||
| 1507 | """ |
|
1511 | """ | |
| 1508 | Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura |
|
1512 | Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura | |
| 1509 | de los datos siempre se realiza por bloques. |
|
1513 | de los datos siempre se realiza por bloques. | |
| 1510 | """ |
|
1514 | """ | |
| 1511 |
|
1515 | |||
| 1512 | blockIndex = 0 |
|
1516 | blockIndex = 0 | |
| 1513 |
|
1517 | |||
| 1514 | path = None |
|
1518 | path = None | |
| 1515 |
|
1519 | |||
| 1516 | setFile = None |
|
1520 | setFile = None | |
| 1517 |
|
1521 | |||
| 1518 | profilesPerBlock = None |
|
1522 | profilesPerBlock = None | |
| 1519 |
|
1523 | |||
| 1520 | blocksPerFile = None |
|
1524 | blocksPerFile = None | |
| 1521 |
|
1525 | |||
| 1522 | nWriteBlocks = 0 |
|
1526 | nWriteBlocks = 0 | |
| 1523 |
|
1527 | |||
| 1524 | fileDate = None |
|
1528 | fileDate = None | |
| 1525 |
|
1529 | |||
| 1526 | def __init__(self, dataOut=None): |
|
1530 | def __init__(self, dataOut=None): | |
| 1527 | raise NotImplementedError |
|
1531 | raise NotImplementedError | |
| 1528 |
|
1532 | |||
| 1529 |
|
1533 | |||
| 1530 | def hasAllDataInBuffer(self): |
|
1534 | def hasAllDataInBuffer(self): | |
| 1531 | raise NotImplementedError |
|
1535 | raise NotImplementedError | |
| 1532 |
|
1536 | |||
| 1533 |
|
1537 | |||
| 1534 | def setBlockDimension(self): |
|
1538 | def setBlockDimension(self): | |
| 1535 | raise NotImplementedError |
|
1539 | raise NotImplementedError | |
| 1536 |
|
1540 | |||
| 1537 |
|
1541 | |||
| 1538 | def writeBlock(self): |
|
1542 | def writeBlock(self): | |
| 1539 | raise NotImplementedError |
|
1543 | raise NotImplementedError | |
| 1540 |
|
1544 | |||
| 1541 |
|
1545 | |||
| 1542 | def putData(self): |
|
1546 | def putData(self): | |
| 1543 | raise NotImplementedError |
|
1547 | raise NotImplementedError | |
| 1544 |
|
1548 | |||
| 1545 |
|
1549 | |||
| 1546 | def getProcessFlags(self): |
|
1550 | def getProcessFlags(self): | |
| 1547 |
|
1551 | |||
| 1548 | processFlags = 0 |
|
1552 | processFlags = 0 | |
| 1549 |
|
1553 | |||
| 1550 | dtype_index = get_dtype_index(self.dtype) |
|
1554 | dtype_index = get_dtype_index(self.dtype) | |
| 1551 | procflag_dtype = get_procflag_dtype(dtype_index) |
|
1555 | procflag_dtype = get_procflag_dtype(dtype_index) | |
| 1552 |
|
1556 | |||
| 1553 | processFlags += procflag_dtype |
|
1557 | processFlags += procflag_dtype | |
| 1554 |
|
1558 | |||
| 1555 | if self.dataOut.flagDecodeData: |
|
1559 | if self.dataOut.flagDecodeData: | |
| 1556 | processFlags += PROCFLAG.DECODE_DATA |
|
1560 | processFlags += PROCFLAG.DECODE_DATA | |
| 1557 |
|
1561 | |||
| 1558 | if self.dataOut.flagDeflipData: |
|
1562 | if self.dataOut.flagDeflipData: | |
| 1559 | processFlags += PROCFLAG.DEFLIP_DATA |
|
1563 | processFlags += PROCFLAG.DEFLIP_DATA | |
| 1560 |
|
1564 | |||
| 1561 | if self.dataOut.code is not None: |
|
1565 | if self.dataOut.code is not None: | |
| 1562 | processFlags += PROCFLAG.DEFINE_PROCESS_CODE |
|
1566 | processFlags += PROCFLAG.DEFINE_PROCESS_CODE | |
| 1563 |
|
1567 | |||
| 1564 | if self.dataOut.nCohInt > 1: |
|
1568 | if self.dataOut.nCohInt > 1: | |
| 1565 | processFlags += PROCFLAG.COHERENT_INTEGRATION |
|
1569 | processFlags += PROCFLAG.COHERENT_INTEGRATION | |
| 1566 |
|
1570 | |||
| 1567 | if self.dataOut.type == "Spectra": |
|
1571 | if self.dataOut.type == "Spectra": | |
| 1568 | if self.dataOut.nIncohInt > 1: |
|
1572 | if self.dataOut.nIncohInt > 1: | |
| 1569 | processFlags += PROCFLAG.INCOHERENT_INTEGRATION |
|
1573 | processFlags += PROCFLAG.INCOHERENT_INTEGRATION | |
| 1570 |
|
1574 | |||
| 1571 | if self.dataOut.data_dc is not None: |
|
1575 | if self.dataOut.data_dc is not None: | |
| 1572 | processFlags += PROCFLAG.SAVE_CHANNELS_DC |
|
1576 | processFlags += PROCFLAG.SAVE_CHANNELS_DC | |
| 1573 |
|
1577 | |||
| 1574 | if self.dataOut.flagShiftFFT: |
|
1578 | if self.dataOut.flagShiftFFT: | |
| 1575 | processFlags += PROCFLAG.SHIFT_FFT_DATA |
|
1579 | processFlags += PROCFLAG.SHIFT_FFT_DATA | |
| 1576 |
|
1580 | |||
| 1577 | return processFlags |
|
1581 | return processFlags | |
| 1578 |
|
1582 | |||
| 1579 | def setBasicHeader(self): |
|
1583 | def setBasicHeader(self): | |
| 1580 |
|
1584 | |||
| 1581 | self.basicHeaderObj.size = self.basicHeaderSize #bytes |
|
1585 | self.basicHeaderObj.size = self.basicHeaderSize #bytes | |
| 1582 | self.basicHeaderObj.version = self.versionFile |
|
1586 | self.basicHeaderObj.version = self.versionFile | |
| 1583 | self.basicHeaderObj.dataBlock = self.nTotalBlocks |
|
1587 | self.basicHeaderObj.dataBlock = self.nTotalBlocks | |
| 1584 |
|
1588 | |||
| 1585 | utc = numpy.floor(self.dataOut.utctime) |
|
1589 | utc = numpy.floor(self.dataOut.utctime) | |
| 1586 | milisecond = (self.dataOut.utctime - utc)* 1000.0 |
|
1590 | milisecond = (self.dataOut.utctime - utc)* 1000.0 | |
| 1587 |
|
1591 | |||
| 1588 | self.basicHeaderObj.utc = utc |
|
1592 | self.basicHeaderObj.utc = utc | |
| 1589 | self.basicHeaderObj.miliSecond = milisecond |
|
1593 | self.basicHeaderObj.miliSecond = milisecond | |
| 1590 | self.basicHeaderObj.timeZone = self.dataOut.timeZone |
|
1594 | self.basicHeaderObj.timeZone = self.dataOut.timeZone | |
| 1591 | self.basicHeaderObj.dstFlag = self.dataOut.dstFlag |
|
1595 | self.basicHeaderObj.dstFlag = self.dataOut.dstFlag | |
| 1592 | self.basicHeaderObj.errorCount = self.dataOut.errorCount |
|
1596 | self.basicHeaderObj.errorCount = self.dataOut.errorCount | |
| 1593 |
|
1597 | |||
| 1594 | def setFirstHeader(self): |
|
1598 | def setFirstHeader(self): | |
| 1595 | """ |
|
1599 | """ | |
| 1596 | Obtiene una copia del First Header |
|
1600 | Obtiene una copia del First Header | |
| 1597 |
|
1601 | |||
| 1598 | Affected: |
|
1602 | Affected: | |
| 1599 |
|
1603 | |||
| 1600 | self.basicHeaderObj |
|
1604 | self.basicHeaderObj | |
| 1601 | self.systemHeaderObj |
|
1605 | self.systemHeaderObj | |
| 1602 | self.radarControllerHeaderObj |
|
1606 | self.radarControllerHeaderObj | |
| 1603 | self.processingHeaderObj self. |
|
1607 | self.processingHeaderObj self. | |
| 1604 |
|
1608 | |||
| 1605 | Return: |
|
1609 | Return: | |
| 1606 | None |
|
1610 | None | |
| 1607 | """ |
|
1611 | """ | |
| 1608 |
|
1612 | |||
| 1609 | raise NotImplementedError |
|
1613 | raise NotImplementedError | |
| 1610 |
|
1614 | |||
| 1611 | def __writeFirstHeader(self): |
|
1615 | def __writeFirstHeader(self): | |
| 1612 | """ |
|
1616 | """ | |
| 1613 | Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader) |
|
1617 | Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader) | |
| 1614 |
|
1618 | |||
| 1615 | Affected: |
|
1619 | Affected: | |
| 1616 | __dataType |
|
1620 | __dataType | |
| 1617 |
|
1621 | |||
| 1618 | Return: |
|
1622 | Return: | |
| 1619 | None |
|
1623 | None | |
| 1620 | """ |
|
1624 | """ | |
| 1621 |
|
1625 | |||
| 1622 | # CALCULAR PARAMETROS |
|
1626 | # CALCULAR PARAMETROS | |
| 1623 |
|
1627 | |||
| 1624 | sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size |
|
1628 | sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size | |
| 1625 | self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader |
|
1629 | self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader | |
| 1626 |
|
1630 | |||
| 1627 | self.basicHeaderObj.write(self.fp) |
|
1631 | self.basicHeaderObj.write(self.fp) | |
| 1628 | self.systemHeaderObj.write(self.fp) |
|
1632 | self.systemHeaderObj.write(self.fp) | |
| 1629 | self.radarControllerHeaderObj.write(self.fp) |
|
1633 | self.radarControllerHeaderObj.write(self.fp) | |
| 1630 | self.processingHeaderObj.write(self.fp) |
|
1634 | self.processingHeaderObj.write(self.fp) | |
| 1631 |
|
1635 | |||
| 1632 | def __setNewBlock(self): |
|
1636 | def __setNewBlock(self): | |
| 1633 | """ |
|
1637 | """ | |
| 1634 | Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header |
|
1638 | Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header | |
| 1635 |
|
1639 | |||
| 1636 | Return: |
|
1640 | Return: | |
| 1637 | 0 : si no pudo escribir nada |
|
1641 | 0 : si no pudo escribir nada | |
| 1638 | 1 : Si escribio el Basic el First Header |
|
1642 | 1 : Si escribio el Basic el First Header | |
| 1639 | """ |
|
1643 | """ | |
| 1640 | if self.fp == None: |
|
1644 | if self.fp == None: | |
| 1641 | self.setNextFile() |
|
1645 | self.setNextFile() | |
| 1642 |
|
1646 | |||
| 1643 | if self.flagIsNewFile: |
|
1647 | if self.flagIsNewFile: | |
| 1644 | return 1 |
|
1648 | return 1 | |
| 1645 |
|
1649 | |||
| 1646 | if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile: |
|
1650 | if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile: | |
| 1647 | self.basicHeaderObj.write(self.fp) |
|
1651 | self.basicHeaderObj.write(self.fp) | |
| 1648 | return 1 |
|
1652 | return 1 | |
| 1649 |
|
1653 | |||
| 1650 | if not( self.setNextFile() ): |
|
1654 | if not( self.setNextFile() ): | |
| 1651 | return 0 |
|
1655 | return 0 | |
| 1652 |
|
1656 | |||
| 1653 | return 1 |
|
1657 | return 1 | |
| 1654 |
|
1658 | |||
| 1655 |
|
1659 | |||
| 1656 | def writeNextBlock(self): |
|
1660 | def writeNextBlock(self): | |
| 1657 | """ |
|
1661 | """ | |
| 1658 | Selecciona el bloque siguiente de datos y los escribe en un file |
|
1662 | Selecciona el bloque siguiente de datos y los escribe en un file | |
| 1659 |
|
1663 | |||
| 1660 | Return: |
|
1664 | Return: | |
| 1661 | 0 : Si no hizo pudo escribir el bloque de datos |
|
1665 | 0 : Si no hizo pudo escribir el bloque de datos | |
| 1662 | 1 : Si no pudo escribir el bloque de datos |
|
1666 | 1 : Si no pudo escribir el bloque de datos | |
| 1663 | """ |
|
1667 | """ | |
| 1664 | if not( self.__setNewBlock() ): |
|
1668 | if not( self.__setNewBlock() ): | |
| 1665 | return 0 |
|
1669 | return 0 | |
| 1666 |
|
1670 | |||
| 1667 | self.writeBlock() |
|
1671 | self.writeBlock() | |
| 1668 |
|
1672 | |||
| 1669 | print "[Writing] Block No. %d/%d" %(self.blockIndex, |
|
1673 | print "[Writing] Block No. %d/%d" %(self.blockIndex, | |
| 1670 | self.processingHeaderObj.dataBlocksPerFile) |
|
1674 | self.processingHeaderObj.dataBlocksPerFile) | |
| 1671 |
|
1675 | |||
| 1672 | return 1 |
|
1676 | return 1 | |
| 1673 |
|
1677 | |||
| 1674 | def setNextFile(self): |
|
1678 | def setNextFile(self): | |
| 1675 | """ |
|
1679 | """ | |
| 1676 | Determina el siguiente file que sera escrito |
|
1680 | Determina el siguiente file que sera escrito | |
| 1677 |
|
1681 | |||
| 1678 | Affected: |
|
1682 | Affected: | |
| 1679 | self.filename |
|
1683 | self.filename | |
| 1680 | self.subfolder |
|
1684 | self.subfolder | |
| 1681 | self.fp |
|
1685 | self.fp | |
| 1682 | self.setFile |
|
1686 | self.setFile | |
| 1683 | self.flagIsNewFile |
|
1687 | self.flagIsNewFile | |
| 1684 |
|
1688 | |||
| 1685 | Return: |
|
1689 | Return: | |
| 1686 | 0 : Si el archivo no puede ser escrito |
|
1690 | 0 : Si el archivo no puede ser escrito | |
| 1687 | 1 : Si el archivo esta listo para ser escrito |
|
1691 | 1 : Si el archivo esta listo para ser escrito | |
| 1688 | """ |
|
1692 | """ | |
| 1689 | ext = self.ext |
|
1693 | ext = self.ext | |
| 1690 | path = self.path |
|
1694 | path = self.path | |
| 1691 |
|
1695 | |||
| 1692 | if self.fp != None: |
|
1696 | if self.fp != None: | |
| 1693 | self.fp.close() |
|
1697 | self.fp.close() | |
| 1694 |
|
1698 | |||
| 1695 | timeTuple = time.localtime( self.dataOut.utctime) |
|
1699 | timeTuple = time.localtime( self.dataOut.utctime) | |
| 1696 | subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) |
|
1700 | subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) | |
| 1697 |
|
1701 | |||
| 1698 | fullpath = os.path.join( path, subfolder ) |
|
1702 | fullpath = os.path.join( path, subfolder ) | |
| 1699 | setFile = self.setFile |
|
1703 | setFile = self.setFile | |
| 1700 |
|
1704 | |||
| 1701 | if not( os.path.exists(fullpath) ): |
|
1705 | if not( os.path.exists(fullpath) ): | |
| 1702 | os.mkdir(fullpath) |
|
1706 | os.mkdir(fullpath) | |
| 1703 | setFile = -1 #inicializo mi contador de seteo |
|
1707 | setFile = -1 #inicializo mi contador de seteo | |
| 1704 | else: |
|
1708 | else: | |
| 1705 | filesList = os.listdir( fullpath ) |
|
1709 | filesList = os.listdir( fullpath ) | |
| 1706 | if len( filesList ) > 0: |
|
1710 | if len( filesList ) > 0: | |
| 1707 | filesList = sorted( filesList, key=str.lower ) |
|
1711 | filesList = sorted( filesList, key=str.lower ) | |
| 1708 | filen = filesList[-1] |
|
1712 | filen = filesList[-1] | |
| 1709 | # el filename debera tener el siguiente formato |
|
1713 | # el filename debera tener el siguiente formato | |
| 1710 | # 0 1234 567 89A BCDE (hex) |
|
1714 | # 0 1234 567 89A BCDE (hex) | |
| 1711 | # x YYYY DDD SSS .ext |
|
1715 | # x YYYY DDD SSS .ext | |
| 1712 | if isNumber( filen[8:11] ): |
|
1716 | if isNumber( filen[8:11] ): | |
| 1713 | setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file |
|
1717 | setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file | |
| 1714 | else: |
|
1718 | else: | |
| 1715 | setFile = -1 |
|
1719 | setFile = -1 | |
| 1716 | else: |
|
1720 | else: | |
| 1717 | setFile = -1 #inicializo mi contador de seteo |
|
1721 | setFile = -1 #inicializo mi contador de seteo | |
| 1718 |
|
1722 | |||
| 1719 | setFile += 1 |
|
1723 | setFile += 1 | |
| 1720 |
|
1724 | |||
| 1721 | #If this is a new day it resets some values |
|
1725 | #If this is a new day it resets some values | |
| 1722 | if self.dataOut.datatime.date() > self.fileDate: |
|
1726 | if self.dataOut.datatime.date() > self.fileDate: | |
| 1723 | setFile = 0 |
|
1727 | setFile = 0 | |
| 1724 | self.nTotalBlocks = 0 |
|
1728 | self.nTotalBlocks = 0 | |
| 1725 |
|
1729 | |||
| 1726 | filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext ) |
|
1730 | filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext ) | |
| 1727 |
|
1731 | |||
| 1728 | filename = os.path.join( path, subfolder, filen ) |
|
1732 | filename = os.path.join( path, subfolder, filen ) | |
| 1729 |
|
1733 | |||
| 1730 | fp = open( filename,'wb' ) |
|
1734 | fp = open( filename,'wb' ) | |
| 1731 |
|
1735 | |||
| 1732 | self.blockIndex = 0 |
|
1736 | self.blockIndex = 0 | |
| 1733 |
|
1737 | |||
| 1734 | #guardando atributos |
|
1738 | #guardando atributos | |
| 1735 | self.filename = filename |
|
1739 | self.filename = filename | |
| 1736 | self.subfolder = subfolder |
|
1740 | self.subfolder = subfolder | |
| 1737 | self.fp = fp |
|
1741 | self.fp = fp | |
| 1738 | self.setFile = setFile |
|
1742 | self.setFile = setFile | |
| 1739 | self.flagIsNewFile = 1 |
|
1743 | self.flagIsNewFile = 1 | |
| 1740 | self.fileDate = self.dataOut.datatime.date() |
|
1744 | self.fileDate = self.dataOut.datatime.date() | |
| 1741 |
|
1745 | |||
| 1742 | self.setFirstHeader() |
|
1746 | self.setFirstHeader() | |
| 1743 |
|
1747 | |||
| 1744 | print '[Writing] Opening file: %s'%self.filename |
|
1748 | print '[Writing] Opening file: %s'%self.filename | |
| 1745 |
|
1749 | |||
| 1746 | self.__writeFirstHeader() |
|
1750 | self.__writeFirstHeader() | |
| 1747 |
|
1751 | |||
| 1748 | return 1 |
|
1752 | return 1 | |
| 1749 |
|
1753 | |||
| 1750 | def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4): |
|
1754 | def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4): | |
| 1751 | """ |
|
1755 | """ | |
| 1752 | Setea el tipo de formato en la cual sera guardada la data y escribe el First Header |
|
1756 | Setea el tipo de formato en la cual sera guardada la data y escribe el First Header | |
| 1753 |
|
1757 | |||
| 1754 | Inputs: |
|
1758 | Inputs: | |
| 1755 | path : directory where data will be saved |
|
1759 | path : directory where data will be saved | |
| 1756 | profilesPerBlock : number of profiles per block |
|
1760 | profilesPerBlock : number of profiles per block | |
| 1757 | set : initial file set |
|
1761 | set : initial file set | |
| 1758 | datatype : An integer number that defines data type: |
|
1762 | datatype : An integer number that defines data type: | |
| 1759 | 0 : int8 (1 byte) |
|
1763 | 0 : int8 (1 byte) | |
| 1760 | 1 : int16 (2 bytes) |
|
1764 | 1 : int16 (2 bytes) | |
| 1761 | 2 : int32 (4 bytes) |
|
1765 | 2 : int32 (4 bytes) | |
| 1762 | 3 : int64 (8 bytes) |
|
1766 | 3 : int64 (8 bytes) | |
| 1763 | 4 : float32 (4 bytes) |
|
1767 | 4 : float32 (4 bytes) | |
| 1764 | 5 : double64 (8 bytes) |
|
1768 | 5 : double64 (8 bytes) | |
| 1765 |
|
1769 | |||
| 1766 | Return: |
|
1770 | Return: | |
| 1767 | 0 : Si no realizo un buen seteo |
|
1771 | 0 : Si no realizo un buen seteo | |
| 1768 | 1 : Si realizo un buen seteo |
|
1772 | 1 : Si realizo un buen seteo | |
| 1769 | """ |
|
1773 | """ | |
| 1770 |
|
1774 | |||
| 1771 | if ext == None: |
|
1775 | if ext == None: | |
| 1772 | ext = self.ext |
|
1776 | ext = self.ext | |
| 1773 |
|
1777 | |||
| 1774 | self.ext = ext.lower() |
|
1778 | self.ext = ext.lower() | |
| 1775 |
|
1779 | |||
| 1776 | self.path = path |
|
1780 | self.path = path | |
| 1777 |
|
1781 | |||
| 1778 | if set is None: |
|
1782 | if set is None: | |
| 1779 | self.setFile = -1 |
|
1783 | self.setFile = -1 | |
| 1780 | else: |
|
1784 | else: | |
| 1781 | self.setFile = set - 1 |
|
1785 | self.setFile = set - 1 | |
| 1782 |
|
1786 | |||
| 1783 | self.blocksPerFile = blocksPerFile |
|
1787 | self.blocksPerFile = blocksPerFile | |
| 1784 |
|
1788 | |||
| 1785 | self.profilesPerBlock = profilesPerBlock |
|
1789 | self.profilesPerBlock = profilesPerBlock | |
| 1786 |
|
1790 | |||
| 1787 | self.dataOut = dataOut |
|
1791 | self.dataOut = dataOut | |
| 1788 | self.fileDate = self.dataOut.datatime.date() |
|
1792 | self.fileDate = self.dataOut.datatime.date() | |
| 1789 | #By default |
|
1793 | #By default | |
| 1790 | self.dtype = self.dataOut.dtype |
|
1794 | self.dtype = self.dataOut.dtype | |
| 1791 |
|
1795 | |||
| 1792 | if datatype is not None: |
|
1796 | if datatype is not None: | |
| 1793 | self.dtype = get_numpy_dtype(datatype) |
|
1797 | self.dtype = get_numpy_dtype(datatype) | |
| 1794 |
|
1798 | |||
| 1795 | if not(self.setNextFile()): |
|
1799 | if not(self.setNextFile()): | |
| 1796 | print "[Writing] There isn't a next file" |
|
1800 | print "[Writing] There isn't a next file" | |
| 1797 | return 0 |
|
1801 | return 0 | |
| 1798 |
|
1802 | |||
| 1799 | self.setBlockDimension() |
|
1803 | self.setBlockDimension() | |
| 1800 |
|
1804 | |||
| 1801 | return 1 |
|
1805 | return 1 | |
| 1802 |
|
1806 | |||
| 1803 | def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs): |
|
1807 | def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs): | |
| 1804 |
|
1808 | |||
| 1805 | if not(self.isConfig): |
|
1809 | if not(self.isConfig): | |
| 1806 |
|
1810 | |||
| 1807 | self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs) |
|
1811 | self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs) | |
| 1808 | self.isConfig = True |
|
1812 | self.isConfig = True | |
| 1809 |
|
1813 | |||
| 1810 | self.putData() |
|
1814 | self.putData() | |
| @@ -1,644 +1,651 | |||||
| 1 | ''' |
|
1 | ''' | |
| 2 | Created on Jul 2, 2014 |
|
2 | Created on Jul 2, 2014 | |
| 3 |
|
3 | |||
| 4 | @author: roj-idl71 |
|
4 | @author: roj-idl71 | |
| 5 | ''' |
|
5 | ''' | |
| 6 |
|
6 | |||
| 7 | import numpy |
|
7 | import numpy | |
| 8 |
|
8 | |||
| 9 | from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter |
|
9 | from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter | |
| 10 | from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation |
|
10 | from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation | |
| 11 | from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader |
|
11 | from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader | |
| 12 | from schainpy.model.data.jrodata import Voltage |
|
12 | from schainpy.model.data.jrodata import Voltage | |
| 13 | import zmq |
|
13 | import zmq | |
| 14 | # from _sha import blocksize |
|
14 | # from _sha import blocksize | |
| 15 |
|
15 | |||
| 16 | class VoltageReader(JRODataReader, ProcessingUnit): |
|
16 | class VoltageReader(JRODataReader, ProcessingUnit): | |
| 17 | """ |
|
17 | """ | |
| 18 | Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura |
|
18 | Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura | |
| 19 | de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones: |
|
19 | de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones: | |
| 20 | perfiles*alturas*canales) son almacenados en la variable "buffer". |
|
20 | perfiles*alturas*canales) son almacenados en la variable "buffer". | |
| 21 |
|
21 | |||
| 22 | perfiles * alturas * canales |
|
22 | perfiles * alturas * canales | |
| 23 |
|
23 | |||
| 24 | Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader, |
|
24 | Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader, | |
| 25 | RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la |
|
25 | RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la | |
| 26 | cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de |
|
26 | cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de | |
| 27 | datos desde el "buffer" cada vez que se ejecute el metodo "getData". |
|
27 | datos desde el "buffer" cada vez que se ejecute el metodo "getData". | |
| 28 |
|
28 | |||
| 29 | Example: |
|
29 | Example: | |
| 30 |
|
30 | |||
| 31 | dpath = "/home/myuser/data" |
|
31 | dpath = "/home/myuser/data" | |
| 32 |
|
32 | |||
| 33 | startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0) |
|
33 | startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0) | |
| 34 |
|
34 | |||
| 35 | endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0) |
|
35 | endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0) | |
| 36 |
|
36 | |||
| 37 | readerObj = VoltageReader() |
|
37 | readerObj = VoltageReader() | |
| 38 |
|
38 | |||
| 39 | readerObj.setup(dpath, startTime, endTime) |
|
39 | readerObj.setup(dpath, startTime, endTime) | |
| 40 |
|
40 | |||
| 41 | while(True): |
|
41 | while(True): | |
| 42 |
|
42 | |||
| 43 | #to get one profile |
|
43 | #to get one profile | |
| 44 | profile = readerObj.getData() |
|
44 | profile = readerObj.getData() | |
| 45 |
|
45 | |||
| 46 | #print the profile |
|
46 | #print the profile | |
| 47 | print profile |
|
47 | print profile | |
| 48 |
|
48 | |||
| 49 | #If you want to see all datablock |
|
49 | #If you want to see all datablock | |
| 50 | print readerObj.datablock |
|
50 | print readerObj.datablock | |
| 51 |
|
51 | |||
| 52 | if readerObj.flagNoMoreFiles: |
|
52 | if readerObj.flagNoMoreFiles: | |
| 53 | break |
|
53 | break | |
| 54 |
|
54 | |||
| 55 | """ |
|
55 | """ | |
| 56 |
|
56 | |||
| 57 | ext = ".r" |
|
57 | ext = ".r" | |
| 58 |
|
58 | |||
| 59 | optchar = "D" |
|
59 | optchar = "D" | |
| 60 | dataOut = None |
|
60 | dataOut = None | |
| 61 |
|
61 | |||
| 62 | def __init__(self, **kwargs): |
|
62 | def __init__(self, **kwargs): | |
| 63 | """ |
|
63 | """ | |
| 64 | Inicializador de la clase VoltageReader para la lectura de datos de voltage. |
|
64 | Inicializador de la clase VoltageReader para la lectura de datos de voltage. | |
| 65 |
|
65 | |||
| 66 | Input: |
|
66 | Input: | |
| 67 | dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para |
|
67 | dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para | |
| 68 | almacenar un perfil de datos cada vez que se haga un requerimiento |
|
68 | almacenar un perfil de datos cada vez que se haga un requerimiento | |
| 69 | (getData). El perfil sera obtenido a partir del buffer de datos, |
|
69 | (getData). El perfil sera obtenido a partir del buffer de datos, | |
| 70 | si el buffer esta vacio se hara un nuevo proceso de lectura de un |
|
70 | si el buffer esta vacio se hara un nuevo proceso de lectura de un | |
| 71 | bloque de datos. |
|
71 | bloque de datos. | |
| 72 | Si este parametro no es pasado se creara uno internamente. |
|
72 | Si este parametro no es pasado se creara uno internamente. | |
| 73 |
|
73 | |||
| 74 | Variables afectadas: |
|
74 | Variables afectadas: | |
| 75 | self.dataOut |
|
75 | self.dataOut | |
| 76 |
|
76 | |||
| 77 | Return: |
|
77 | Return: | |
| 78 | None |
|
78 | None | |
| 79 | """ |
|
79 | """ | |
| 80 |
|
80 | |||
| 81 | ProcessingUnit.__init__(self, **kwargs) |
|
81 | ProcessingUnit.__init__(self, **kwargs) | |
| 82 |
|
82 | |||
| 83 | self.isConfig = False |
|
83 | self.isConfig = False | |
| 84 |
|
84 | |||
| 85 | self.datablock = None |
|
85 | self.datablock = None | |
| 86 |
|
86 | |||
| 87 | self.utc = 0 |
|
87 | self.utc = 0 | |
| 88 |
|
88 | |||
| 89 | self.ext = ".r" |
|
89 | self.ext = ".r" | |
| 90 |
|
90 | |||
| 91 | self.optchar = "D" |
|
91 | self.optchar = "D" | |
| 92 |
|
92 | |||
| 93 | self.basicHeaderObj = BasicHeader(LOCALTIME) |
|
93 | self.basicHeaderObj = BasicHeader(LOCALTIME) | |
| 94 |
|
94 | |||
| 95 | self.systemHeaderObj = SystemHeader() |
|
95 | self.systemHeaderObj = SystemHeader() | |
| 96 |
|
96 | |||
| 97 | self.radarControllerHeaderObj = RadarControllerHeader() |
|
97 | self.radarControllerHeaderObj = RadarControllerHeader() | |
| 98 |
|
98 | |||
| 99 | self.processingHeaderObj = ProcessingHeader() |
|
99 | self.processingHeaderObj = ProcessingHeader() | |
| 100 |
|
100 | |||
| 101 | self.online = 0 |
|
101 | self.online = 0 | |
| 102 |
|
102 | |||
| 103 | self.fp = None |
|
103 | self.fp = None | |
| 104 |
|
104 | |||
| 105 | self.idFile = None |
|
105 | self.idFile = None | |
| 106 |
|
106 | |||
| 107 | self.dtype = None |
|
107 | self.dtype = None | |
| 108 |
|
108 | |||
| 109 | self.fileSizeByHeader = None |
|
109 | self.fileSizeByHeader = None | |
| 110 |
|
110 | |||
| 111 | self.filenameList = [] |
|
111 | self.filenameList = [] | |
| 112 |
|
112 | |||
| 113 | self.filename = None |
|
113 | self.filename = None | |
| 114 |
|
114 | |||
| 115 | self.fileSize = None |
|
115 | self.fileSize = None | |
| 116 |
|
116 | |||
| 117 | self.firstHeaderSize = 0 |
|
117 | self.firstHeaderSize = 0 | |
| 118 |
|
118 | |||
| 119 | self.basicHeaderSize = 24 |
|
119 | self.basicHeaderSize = 24 | |
| 120 |
|
120 | |||
| 121 | self.pathList = [] |
|
121 | self.pathList = [] | |
| 122 |
|
122 | |||
| 123 | self.filenameList = [] |
|
123 | self.filenameList = [] | |
| 124 |
|
124 | |||
| 125 | self.lastUTTime = 0 |
|
125 | self.lastUTTime = 0 | |
| 126 |
|
126 | |||
| 127 | self.maxTimeStep = 30 |
|
127 | self.maxTimeStep = 30 | |
| 128 |
|
128 | |||
| 129 | self.flagNoMoreFiles = 0 |
|
129 | self.flagNoMoreFiles = 0 | |
| 130 |
|
130 | |||
| 131 | self.set = 0 |
|
131 | self.set = 0 | |
| 132 |
|
132 | |||
| 133 | self.path = None |
|
133 | self.path = None | |
| 134 |
|
134 | |||
| 135 | self.profileIndex = 2**32-1 |
|
135 | self.profileIndex = 2**32-1 | |
| 136 |
|
136 | |||
| 137 | self.delay = 3 #seconds |
|
137 | self.delay = 3 #seconds | |
| 138 |
|
138 | |||
| 139 | self.nTries = 3 #quantity tries |
|
139 | self.nTries = 3 #quantity tries | |
| 140 |
|
140 | |||
| 141 | self.nFiles = 3 #number of files for searching |
|
141 | self.nFiles = 3 #number of files for searching | |
| 142 |
|
142 | |||
| 143 | self.nReadBlocks = 0 |
|
143 | self.nReadBlocks = 0 | |
| 144 |
|
144 | |||
| 145 | self.flagIsNewFile = 1 |
|
145 | self.flagIsNewFile = 1 | |
| 146 |
|
146 | |||
| 147 | self.__isFirstTimeOnline = 1 |
|
147 | self.__isFirstTimeOnline = 1 | |
| 148 |
|
148 | |||
| 149 | # self.ippSeconds = 0 |
|
149 | # self.ippSeconds = 0 | |
| 150 |
|
150 | |||
| 151 | self.flagDiscontinuousBlock = 0 |
|
151 | self.flagDiscontinuousBlock = 0 | |
| 152 |
|
152 | |||
| 153 | self.flagIsNewBlock = 0 |
|
153 | self.flagIsNewBlock = 0 | |
| 154 |
|
154 | |||
| 155 | self.nTotalBlocks = 0 |
|
155 | self.nTotalBlocks = 0 | |
| 156 |
|
156 | |||
| 157 | self.blocksize = 0 |
|
157 | self.blocksize = 0 | |
| 158 |
|
158 | |||
| 159 | self.dataOut = self.createObjByDefault() |
|
159 | self.dataOut = self.createObjByDefault() | |
| 160 |
|
160 | |||
| 161 | self.nTxs = 1 |
|
161 | self.nTxs = 1 | |
| 162 |
|
162 | |||
| 163 | self.txIndex = 0 |
|
163 | self.txIndex = 0 | |
| 164 |
|
164 | |||
| 165 | def createObjByDefault(self): |
|
165 | def createObjByDefault(self): | |
| 166 |
|
166 | |||
| 167 | dataObj = Voltage() |
|
167 | dataObj = Voltage() | |
| 168 |
|
168 | |||
| 169 | return dataObj |
|
169 | return dataObj | |
| 170 |
|
170 | |||
| 171 | def __hasNotDataInBuffer(self): |
|
171 | def __hasNotDataInBuffer(self): | |
| 172 |
|
172 | |||
| 173 | if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs: |
|
173 | if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs: | |
| 174 | return 1 |
|
174 | return 1 | |
| 175 |
|
175 | |||
| 176 | return 0 |
|
176 | return 0 | |
| 177 |
|
177 | |||
| 178 |
|
178 | |||
| 179 | def getBlockDimension(self): |
|
179 | def getBlockDimension(self): | |
| 180 | """ |
|
180 | """ | |
| 181 | Obtiene la cantidad de puntos a leer por cada bloque de datos |
|
181 | Obtiene la cantidad de puntos a leer por cada bloque de datos | |
| 182 |
|
182 | |||
| 183 | Affected: |
|
183 | Affected: | |
| 184 | self.blocksize |
|
184 | self.blocksize | |
| 185 |
|
185 | |||
| 186 | Return: |
|
186 | Return: | |
| 187 | None |
|
187 | None | |
| 188 | """ |
|
188 | """ | |
| 189 | pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels |
|
189 | pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels | |
| 190 | self.blocksize = pts2read |
|
190 | self.blocksize = pts2read | |
| 191 |
|
191 | |||
|
|
192 | ||||
| 192 |
|
193 | |||
| 193 | def readBlock(self): |
|
194 | def readBlock(self): | |
| 194 | """ |
|
195 | """ | |
| 195 | readBlock lee el bloque de datos desde la posicion actual del puntero del archivo |
|
196 | readBlock lee el bloque de datos desde la posicion actual del puntero del archivo | |
| 196 | (self.fp) y actualiza todos los parametros relacionados al bloque de datos |
|
197 | (self.fp) y actualiza todos los parametros relacionados al bloque de datos | |
| 197 | (metadata + data). La data leida es almacenada en el buffer y el contador del buffer |
|
198 | (metadata + data). La data leida es almacenada en el buffer y el contador del buffer | |
| 198 | es seteado a 0 |
|
199 | es seteado a 0 | |
| 199 |
|
200 | |||
| 200 | Inputs: |
|
201 | Inputs: | |
| 201 | None |
|
202 | None | |
| 202 |
|
203 | |||
| 203 | Return: |
|
204 | Return: | |
| 204 | None |
|
205 | None | |
| 205 |
|
206 | |||
| 206 | Affected: |
|
207 | Affected: | |
| 207 | self.profileIndex |
|
208 | self.profileIndex | |
| 208 | self.datablock |
|
209 | self.datablock | |
| 209 | self.flagIsNewFile |
|
210 | self.flagIsNewFile | |
| 210 | self.flagIsNewBlock |
|
211 | self.flagIsNewBlock | |
| 211 | self.nTotalBlocks |
|
212 | self.nTotalBlocks | |
| 212 |
|
213 | |||
| 213 | Exceptions: |
|
214 | Exceptions: | |
| 214 | Si un bloque leido no es un bloque valido |
|
215 | Si un bloque leido no es un bloque valido | |
| 215 | """ |
|
216 | """ | |
| 216 | current_pointer_location = self.fp.tell() |
|
217 | ||
| 217 | junk = numpy.fromfile( self.fp, self.dtype, self.blocksize ) |
|
218 | print 'READ BLOCK' | |
|
|
219 | if self.server is not None: | |||
|
|
220 | self.zBlock = self.receiver.recv() | |||
|
|
221 | self.zHeader = self.zBlock[:24] | |||
|
|
222 | self.zDataBlock = self.zBlock[24:] | |||
|
|
223 | junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')])) | |||
|
|
224 | self.processingHeaderObj.profilesPerBlock = 240 | |||
|
|
225 | self.processingHeaderObj.nHeights = 248 | |||
|
|
226 | self.systemHeaderObj.nChannels | |||
|
|
227 | else: | |||
|
|
228 | current_pointer_location = self.fp.tell() | |||
|
|
229 | junk = numpy.fromfile( self.fp, self.dtype, self.blocksize ) | |||
| 218 |
|
230 | |||
| 219 | try: |
|
231 | try: | |
| 220 | junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) ) |
|
232 | junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) ) | |
|
|
233 | print'junked' | |||
| 221 | except: |
|
234 | except: | |
| 222 | #print "The read block (%3d) has not enough data" %self.nReadBlocks |
|
235 | #print "The read block (%3d) has not enough data" %self.nReadBlocks | |
| 223 |
|
236 | |||
| 224 | if self.waitDataBlock(pointer_location=current_pointer_location): |
|
237 | if self.waitDataBlock(pointer_location=current_pointer_location): | |
| 225 | junk = numpy.fromfile( self.fp, self.dtype, self.blocksize ) |
|
238 | junk = numpy.fromfile( self.fp, self.dtype, self.blocksize ) | |
| 226 | junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) ) |
|
239 | junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) ) | |
| 227 | # return 0 |
|
240 | # return 0 | |
| 228 |
|
241 | |||
| 229 | #Dimensions : nChannels, nProfiles, nSamples |
|
242 | #Dimensions : nChannels, nProfiles, nSamples | |
| 230 |
|
243 | |||
| 231 | junk = numpy.transpose(junk, (2,0,1)) |
|
244 | junk = numpy.transpose(junk, (2,0,1)) | |
| 232 | self.datablock = junk['real'] + junk['imag']*1j |
|
245 | self.datablock = junk['real'] + junk['imag']*1j | |
| 233 |
|
246 | |||
| 234 | self.profileIndex = 0 |
|
247 | self.profileIndex = 0 | |
| 235 |
|
248 | |||
| 236 | self.flagIsNewFile = 0 |
|
249 | self.flagIsNewFile = 0 | |
| 237 | self.flagIsNewBlock = 1 |
|
250 | self.flagIsNewBlock = 1 | |
| 238 |
|
251 | |||
| 239 | self.nTotalBlocks += 1 |
|
252 | self.nTotalBlocks += 1 | |
| 240 | self.nReadBlocks += 1 |
|
253 | self.nReadBlocks += 1 | |
| 241 |
|
254 | |||
| 242 | return 1 |
|
255 | return 1 | |
| 243 |
|
256 | |||
| 244 | def getFirstHeader(self): |
|
257 | def getFirstHeader(self): | |
| 245 |
|
258 | |||
| 246 | self.getBasicHeader() |
|
259 | self.getBasicHeader() | |
| 247 |
|
260 | |||
| 248 | self.dataOut.systemHeaderObj = self.systemHeaderObj.copy() |
|
261 | self.dataOut.systemHeaderObj = self.systemHeaderObj.copy() | |
| 249 |
|
262 | |||
| 250 | self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy() |
|
263 | self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy() | |
| 251 |
|
264 | |||
| 252 | if self.nTxs > 1: |
|
265 | if self.nTxs > 1: | |
| 253 | self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs |
|
266 | self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs | |
| 254 |
|
267 | |||
| 255 | #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj. |
|
268 | #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj. | |
| 256 |
|
269 | |||
| 257 | # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt |
|
270 | # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt | |
| 258 | # |
|
271 | # | |
| 259 | # if self.radarControllerHeaderObj.code is not None: |
|
272 | # if self.radarControllerHeaderObj.code is not None: | |
| 260 | # |
|
273 | # | |
| 261 | # self.dataOut.nCode = self.radarControllerHeaderObj.nCode |
|
274 | # self.dataOut.nCode = self.radarControllerHeaderObj.nCode | |
| 262 | # |
|
275 | # | |
| 263 | # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud |
|
276 | # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud | |
| 264 | # |
|
277 | # | |
| 265 | # self.dataOut.code = self.radarControllerHeaderObj.code |
|
278 | # self.dataOut.code = self.radarControllerHeaderObj.code | |
| 266 |
|
279 | |||
| 267 | self.dataOut.dtype = self.dtype |
|
280 | self.dataOut.dtype = self.dtype | |
| 268 |
|
281 | |||
| 269 | self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock |
|
282 | self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock | |
| 270 |
|
283 | |||
| 271 | self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight |
|
284 | self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight | |
| 272 |
|
285 | |||
| 273 | self.dataOut.channelList = range(self.systemHeaderObj.nChannels) |
|
286 | self.dataOut.channelList = range(self.systemHeaderObj.nChannels) | |
| 274 |
|
287 | |||
| 275 | self.dataOut.nCohInt = self.processingHeaderObj.nCohInt |
|
288 | self.dataOut.nCohInt = self.processingHeaderObj.nCohInt | |
| 276 |
|
289 | |||
| 277 | self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada |
|
290 | self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada | |
| 278 |
|
291 | |||
| 279 | self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip |
|
292 | self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip | |
| 280 |
|
293 | |||
| 281 | self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft |
|
294 | self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft | |
| 282 |
|
295 | |||
| 283 | def reshapeData(self): |
|
296 | def reshapeData(self): | |
| 284 |
|
297 | |||
| 285 | if self.nTxs < 0: |
|
298 | if self.nTxs < 0: | |
| 286 | return |
|
299 | return | |
| 287 |
|
300 | |||
| 288 | if self.nTxs == 1: |
|
301 | if self.nTxs == 1: | |
| 289 | return |
|
302 | return | |
| 290 |
|
303 | |||
| 291 | if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0: |
|
304 | if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0: | |
| 292 | raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock) |
|
305 | raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock) | |
| 293 |
|
306 | |||
| 294 | if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0: |
|
307 | if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0: | |
| 295 | raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights) |
|
308 | raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights) | |
| 296 |
|
309 | |||
| 297 | self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs)) |
|
310 | self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs)) | |
| 298 |
|
311 | |||
| 299 | self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs |
|
312 | self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs | |
| 300 | self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight |
|
313 | self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight | |
| 301 | self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs |
|
314 | self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs | |
| 302 |
|
315 | |||
| 303 | return |
|
316 | return | |
| 304 |
|
317 | |||
| 305 |
|
||||
| 306 | def getFromZMQ(self): |
|
|||
| 307 | self.dataOut = self.receiver.recv_pyobj() |
|
|||
| 308 | print '[Receiving] {} - {}'.format(self.dataOut.type, |
|
|||
| 309 | self.dataOut.datatime.ctime()) |
|
|||
| 310 |
|
||||
| 311 | def getData(self): |
|
318 | def getData(self): | |
| 312 | """ |
|
319 | """ | |
| 313 | getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut |
|
320 | getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut | |
| 314 | del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos |
|
321 | del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos | |
| 315 | en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando |
|
322 | en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando | |
| 316 | "readNextBlock" |
|
323 | "readNextBlock" | |
| 317 |
|
324 | |||
| 318 | Ademas incrementa el contador del buffer "self.profileIndex" en 1. |
|
325 | Ademas incrementa el contador del buffer "self.profileIndex" en 1. | |
| 319 |
|
326 | |||
| 320 | Return: |
|
327 | Return: | |
| 321 |
|
328 | |||
| 322 | Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex |
|
329 | Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex | |
| 323 | es igual al total de perfiles leidos desde el archivo. |
|
330 | es igual al total de perfiles leidos desde el archivo. | |
| 324 |
|
331 | |||
| 325 | Si self.getByBlock == False: |
|
332 | Si self.getByBlock == False: | |
| 326 |
|
333 | |||
| 327 | self.dataOut.data = buffer[:, thisProfile, :] |
|
334 | self.dataOut.data = buffer[:, thisProfile, :] | |
| 328 |
|
335 | |||
| 329 | shape = [nChannels, nHeis] |
|
336 | shape = [nChannels, nHeis] | |
| 330 |
|
337 | |||
| 331 | Si self.getByBlock == True: |
|
338 | Si self.getByBlock == True: | |
| 332 |
|
339 | |||
| 333 | self.dataOut.data = buffer[:, :, :] |
|
340 | self.dataOut.data = buffer[:, :, :] | |
| 334 |
|
341 | |||
| 335 | shape = [nChannels, nProfiles, nHeis] |
|
342 | shape = [nChannels, nProfiles, nHeis] | |
| 336 |
|
343 | |||
| 337 | Variables afectadas: |
|
344 | Variables afectadas: | |
| 338 | self.dataOut |
|
345 | self.dataOut | |
| 339 | self.profileIndex |
|
346 | self.profileIndex | |
| 340 |
|
347 | |||
| 341 | Affected: |
|
348 | Affected: | |
| 342 | self.dataOut |
|
349 | self.dataOut | |
| 343 | self.profileIndex |
|
350 | self.profileIndex | |
| 344 | self.flagDiscontinuousBlock |
|
351 | self.flagDiscontinuousBlock | |
| 345 | self.flagIsNewBlock |
|
352 | self.flagIsNewBlock | |
| 346 | """ |
|
353 | """ | |
| 347 |
|
354 | print 1 | ||
| 348 | if self.flagNoMoreFiles: |
|
355 | if self.flagNoMoreFiles: | |
| 349 | self.dataOut.flagNoData = True |
|
356 | self.dataOut.flagNoData = True | |
| 350 | print 'Process finished' |
|
357 | print 'Process finished' | |
| 351 | return 0 |
|
358 | return 0 | |
| 352 |
|
359 | print 2 | ||
| 353 | self.flagDiscontinuousBlock = 0 |
|
360 | self.flagDiscontinuousBlock = 0 | |
| 354 | self.flagIsNewBlock = 0 |
|
361 | self.flagIsNewBlock = 0 | |
| 355 |
|
362 | print 3 | ||
| 356 | if self.__hasNotDataInBuffer(): |
|
363 | if self.__hasNotDataInBuffer(): | |
| 357 |
|
364 | |||
| 358 | if not( self.readNextBlock() ): |
|
365 | if not( self.readNextBlock() ): | |
| 359 | return 0 |
|
366 | return 0 | |
| 360 |
|
367 | |||
| 361 | self.getFirstHeader() |
|
368 | self.getFirstHeader() | |
| 362 |
|
369 | |||
| 363 | self.reshapeData() |
|
370 | self.reshapeData() | |
| 364 |
|
371 | print 4 | ||
| 365 | if self.datablock is None: |
|
372 | if self.datablock is None: | |
| 366 | self.dataOut.flagNoData = True |
|
373 | self.dataOut.flagNoData = True | |
| 367 | return 0 |
|
374 | return 0 | |
| 368 |
|
375 | |||
| 369 | if not self.getByBlock: |
|
376 | if not self.getByBlock: | |
| 370 |
|
377 | |||
| 371 | """ |
|
378 | """ | |
| 372 | Return profile by profile |
|
379 | Return profile by profile | |
| 373 |
|
380 | |||
| 374 | If nTxs > 1 then one profile is divided by nTxs and number of total |
|
381 | If nTxs > 1 then one profile is divided by nTxs and number of total | |
| 375 | blocks is increased by nTxs (nProfiles *= nTxs) |
|
382 | blocks is increased by nTxs (nProfiles *= nTxs) | |
| 376 | """ |
|
383 | """ | |
| 377 | self.dataOut.flagDataAsBlock = False |
|
384 | self.dataOut.flagDataAsBlock = False | |
| 378 | self.dataOut.data = self.datablock[:,self.profileIndex,:] |
|
385 | self.dataOut.data = self.datablock[:,self.profileIndex,:] | |
| 379 | self.dataOut.profileIndex = self.profileIndex |
|
386 | self.dataOut.profileIndex = self.profileIndex | |
| 380 |
|
387 | |||
| 381 | self.profileIndex += 1 |
|
388 | self.profileIndex += 1 | |
| 382 |
|
389 | |||
| 383 | # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles: |
|
390 | # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles: | |
| 384 | # """ |
|
391 | # """ | |
| 385 | # Return all block |
|
392 | # Return all block | |
| 386 | # """ |
|
393 | # """ | |
| 387 | # self.dataOut.flagDataAsBlock = True |
|
394 | # self.dataOut.flagDataAsBlock = True | |
| 388 | # self.dataOut.data = self.datablock |
|
395 | # self.dataOut.data = self.datablock | |
| 389 | # self.dataOut.profileIndex = self.dataOut.nProfiles - 1 |
|
396 | # self.dataOut.profileIndex = self.dataOut.nProfiles - 1 | |
| 390 | # |
|
397 | # | |
| 391 | # self.profileIndex = self.dataOut.nProfiles |
|
398 | # self.profileIndex = self.dataOut.nProfiles | |
| 392 |
|
399 | |||
| 393 | else: |
|
400 | else: | |
| 394 | """ |
|
401 | """ | |
| 395 | Return a block |
|
402 | Return a block | |
| 396 | """ |
|
403 | """ | |
| 397 | if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles |
|
404 | if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles | |
| 398 | if self.selBlocktime != None: |
|
405 | if self.selBlocktime != None: | |
| 399 | if self.dataOut.nCohInt is not None: |
|
406 | if self.dataOut.nCohInt is not None: | |
| 400 | nCohInt = self.dataOut.nCohInt |
|
407 | nCohInt = self.dataOut.nCohInt | |
| 401 | else: |
|
408 | else: | |
| 402 | nCohInt = 1 |
|
409 | nCohInt = 1 | |
| 403 | self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles))) |
|
410 | self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles))) | |
| 404 |
|
411 | |||
| 405 | self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:] |
|
412 | self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:] | |
| 406 | self.profileIndex += self.selBlocksize |
|
413 | self.profileIndex += self.selBlocksize | |
| 407 | datasize = self.dataOut.data.shape[1] |
|
414 | datasize = self.dataOut.data.shape[1] | |
| 408 |
|
415 | |||
| 409 | if datasize < self.selBlocksize: |
|
416 | if datasize < self.selBlocksize: | |
| 410 | buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex') |
|
417 | buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex') | |
| 411 | buffer[:,:datasize,:] = self.dataOut.data |
|
418 | buffer[:,:datasize,:] = self.dataOut.data | |
| 412 |
|
419 | |||
| 413 | while datasize < self.selBlocksize: #Not enough profiles to fill the block |
|
420 | while datasize < self.selBlocksize: #Not enough profiles to fill the block | |
| 414 | if not( self.readNextBlock() ): |
|
421 | if not( self.readNextBlock() ): | |
| 415 | return 0 |
|
422 | return 0 | |
| 416 | self.getFirstHeader() |
|
423 | self.getFirstHeader() | |
| 417 | self.reshapeData() |
|
424 | self.reshapeData() | |
| 418 | if self.datablock is None: |
|
425 | if self.datablock is None: | |
| 419 | self.dataOut.flagNoData = True |
|
426 | self.dataOut.flagNoData = True | |
| 420 | return 0 |
|
427 | return 0 | |
| 421 | #stack data |
|
428 | #stack data | |
| 422 | blockIndex = self.selBlocksize - datasize |
|
429 | blockIndex = self.selBlocksize - datasize | |
| 423 | datablock1 = self.datablock[:,:blockIndex,:] |
|
430 | datablock1 = self.datablock[:,:blockIndex,:] | |
| 424 |
|
431 | |||
| 425 | buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1 |
|
432 | buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1 | |
| 426 | datasize += datablock1.shape[1] |
|
433 | datasize += datablock1.shape[1] | |
| 427 |
|
434 | |||
| 428 | self.dataOut.data = buffer |
|
435 | self.dataOut.data = buffer | |
| 429 | self.profileIndex = blockIndex |
|
436 | self.profileIndex = blockIndex | |
| 430 |
|
437 | |||
| 431 | self.dataOut.flagDataAsBlock = True |
|
438 | self.dataOut.flagDataAsBlock = True | |
| 432 | self.dataOut.nProfiles = self.dataOut.data.shape[1] |
|
439 | self.dataOut.nProfiles = self.dataOut.data.shape[1] | |
| 433 |
|
440 | |||
| 434 | self.dataOut.flagNoData = False |
|
441 | self.dataOut.flagNoData = False | |
| 435 |
|
442 | |||
| 436 | self.getBasicHeader() |
|
443 | self.getBasicHeader() | |
| 437 |
|
444 | |||
| 438 | self.dataOut.realtime = self.online |
|
445 | self.dataOut.realtime = self.online | |
| 439 |
|
446 | |||
| 440 | return self.dataOut.data |
|
447 | return self.dataOut.data | |
| 441 |
|
448 | |||
| 442 | class VoltageWriter(JRODataWriter, Operation): |
|
449 | class VoltageWriter(JRODataWriter, Operation): | |
| 443 | """ |
|
450 | """ | |
| 444 | Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura |
|
451 | Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura | |
| 445 | de los datos siempre se realiza por bloques. |
|
452 | de los datos siempre se realiza por bloques. | |
| 446 | """ |
|
453 | """ | |
| 447 |
|
454 | |||
| 448 | ext = ".r" |
|
455 | ext = ".r" | |
| 449 |
|
456 | |||
| 450 | optchar = "D" |
|
457 | optchar = "D" | |
| 451 |
|
458 | |||
| 452 | shapeBuffer = None |
|
459 | shapeBuffer = None | |
| 453 |
|
460 | |||
| 454 |
|
461 | |||
| 455 | def __init__(self, **kwargs): |
|
462 | def __init__(self, **kwargs): | |
| 456 | """ |
|
463 | """ | |
| 457 | Inicializador de la clase VoltageWriter para la escritura de datos de espectros. |
|
464 | Inicializador de la clase VoltageWriter para la escritura de datos de espectros. | |
| 458 |
|
465 | |||
| 459 | Affected: |
|
466 | Affected: | |
| 460 | self.dataOut |
|
467 | self.dataOut | |
| 461 |
|
468 | |||
| 462 | Return: None |
|
469 | Return: None | |
| 463 | """ |
|
470 | """ | |
| 464 | Operation.__init__(self, **kwargs) |
|
471 | Operation.__init__(self, **kwargs) | |
| 465 |
|
472 | |||
| 466 | self.nTotalBlocks = 0 |
|
473 | self.nTotalBlocks = 0 | |
| 467 |
|
474 | |||
| 468 | self.profileIndex = 0 |
|
475 | self.profileIndex = 0 | |
| 469 |
|
476 | |||
| 470 | self.isConfig = False |
|
477 | self.isConfig = False | |
| 471 |
|
478 | |||
| 472 | self.fp = None |
|
479 | self.fp = None | |
| 473 |
|
480 | |||
| 474 | self.flagIsNewFile = 1 |
|
481 | self.flagIsNewFile = 1 | |
| 475 |
|
482 | |||
| 476 | self.blockIndex = 0 |
|
483 | self.blockIndex = 0 | |
| 477 |
|
484 | |||
| 478 | self.flagIsNewBlock = 0 |
|
485 | self.flagIsNewBlock = 0 | |
| 479 |
|
486 | |||
| 480 | self.setFile = None |
|
487 | self.setFile = None | |
| 481 |
|
488 | |||
| 482 | self.dtype = None |
|
489 | self.dtype = None | |
| 483 |
|
490 | |||
| 484 | self.path = None |
|
491 | self.path = None | |
| 485 |
|
492 | |||
| 486 | self.filename = None |
|
493 | self.filename = None | |
| 487 |
|
494 | |||
| 488 | self.basicHeaderObj = BasicHeader(LOCALTIME) |
|
495 | self.basicHeaderObj = BasicHeader(LOCALTIME) | |
| 489 |
|
496 | |||
| 490 | self.systemHeaderObj = SystemHeader() |
|
497 | self.systemHeaderObj = SystemHeader() | |
| 491 |
|
498 | |||
| 492 | self.radarControllerHeaderObj = RadarControllerHeader() |
|
499 | self.radarControllerHeaderObj = RadarControllerHeader() | |
| 493 |
|
500 | |||
| 494 | self.processingHeaderObj = ProcessingHeader() |
|
501 | self.processingHeaderObj = ProcessingHeader() | |
| 495 |
|
502 | |||
| 496 | def hasAllDataInBuffer(self): |
|
503 | def hasAllDataInBuffer(self): | |
| 497 | if self.profileIndex >= self.processingHeaderObj.profilesPerBlock: |
|
504 | if self.profileIndex >= self.processingHeaderObj.profilesPerBlock: | |
| 498 | return 1 |
|
505 | return 1 | |
| 499 | return 0 |
|
506 | return 0 | |
| 500 |
|
507 | |||
| 501 |
|
508 | |||
| 502 | def setBlockDimension(self): |
|
509 | def setBlockDimension(self): | |
| 503 | """ |
|
510 | """ | |
| 504 | Obtiene las formas dimensionales del los subbloques de datos que componen un bloque |
|
511 | Obtiene las formas dimensionales del los subbloques de datos que componen un bloque | |
| 505 |
|
512 | |||
| 506 | Affected: |
|
513 | Affected: | |
| 507 | self.shape_spc_Buffer |
|
514 | self.shape_spc_Buffer | |
| 508 | self.shape_cspc_Buffer |
|
515 | self.shape_cspc_Buffer | |
| 509 | self.shape_dc_Buffer |
|
516 | self.shape_dc_Buffer | |
| 510 |
|
517 | |||
| 511 | Return: None |
|
518 | Return: None | |
| 512 | """ |
|
519 | """ | |
| 513 | self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock, |
|
520 | self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock, | |
| 514 | self.processingHeaderObj.nHeights, |
|
521 | self.processingHeaderObj.nHeights, | |
| 515 | self.systemHeaderObj.nChannels) |
|
522 | self.systemHeaderObj.nChannels) | |
| 516 |
|
523 | |||
| 517 | self.datablock = numpy.zeros((self.systemHeaderObj.nChannels, |
|
524 | self.datablock = numpy.zeros((self.systemHeaderObj.nChannels, | |
| 518 | self.processingHeaderObj.profilesPerBlock, |
|
525 | self.processingHeaderObj.profilesPerBlock, | |
| 519 | self.processingHeaderObj.nHeights), |
|
526 | self.processingHeaderObj.nHeights), | |
| 520 | dtype=numpy.dtype('complex64')) |
|
527 | dtype=numpy.dtype('complex64')) | |
| 521 |
|
528 | |||
| 522 | def writeBlock(self): |
|
529 | def writeBlock(self): | |
| 523 | """ |
|
530 | """ | |
| 524 | Escribe el buffer en el file designado |
|
531 | Escribe el buffer en el file designado | |
| 525 |
|
532 | |||
| 526 | Affected: |
|
533 | Affected: | |
| 527 | self.profileIndex |
|
534 | self.profileIndex | |
| 528 | self.flagIsNewFile |
|
535 | self.flagIsNewFile | |
| 529 | self.flagIsNewBlock |
|
536 | self.flagIsNewBlock | |
| 530 | self.nTotalBlocks |
|
537 | self.nTotalBlocks | |
| 531 | self.blockIndex |
|
538 | self.blockIndex | |
| 532 |
|
539 | |||
| 533 | Return: None |
|
540 | Return: None | |
| 534 | """ |
|
541 | """ | |
| 535 | data = numpy.zeros( self.shapeBuffer, self.dtype ) |
|
542 | data = numpy.zeros( self.shapeBuffer, self.dtype ) | |
| 536 |
|
543 | |||
| 537 | junk = numpy.transpose(self.datablock, (1,2,0)) |
|
544 | junk = numpy.transpose(self.datablock, (1,2,0)) | |
| 538 |
|
545 | |||
| 539 | data['real'] = junk.real |
|
546 | data['real'] = junk.real | |
| 540 | data['imag'] = junk.imag |
|
547 | data['imag'] = junk.imag | |
| 541 |
|
548 | |||
| 542 | data = data.reshape( (-1) ) |
|
549 | data = data.reshape( (-1) ) | |
| 543 |
|
550 | |||
| 544 | data.tofile( self.fp ) |
|
551 | data.tofile( self.fp ) | |
| 545 |
|
552 | |||
| 546 | self.datablock.fill(0) |
|
553 | self.datablock.fill(0) | |
| 547 |
|
554 | |||
| 548 | self.profileIndex = 0 |
|
555 | self.profileIndex = 0 | |
| 549 | self.flagIsNewFile = 0 |
|
556 | self.flagIsNewFile = 0 | |
| 550 | self.flagIsNewBlock = 1 |
|
557 | self.flagIsNewBlock = 1 | |
| 551 |
|
558 | |||
| 552 | self.blockIndex += 1 |
|
559 | self.blockIndex += 1 | |
| 553 | self.nTotalBlocks += 1 |
|
560 | self.nTotalBlocks += 1 | |
| 554 |
|
561 | |||
| 555 | # print "[Writing] Block = %04d" %self.blockIndex |
|
562 | # print "[Writing] Block = %04d" %self.blockIndex | |
| 556 |
|
563 | |||
| 557 | def putData(self): |
|
564 | def putData(self): | |
| 558 | """ |
|
565 | """ | |
| 559 | Setea un bloque de datos y luego los escribe en un file |
|
566 | Setea un bloque de datos y luego los escribe en un file | |
| 560 |
|
567 | |||
| 561 | Affected: |
|
568 | Affected: | |
| 562 | self.flagIsNewBlock |
|
569 | self.flagIsNewBlock | |
| 563 | self.profileIndex |
|
570 | self.profileIndex | |
| 564 |
|
571 | |||
| 565 | Return: |
|
572 | Return: | |
| 566 | 0 : Si no hay data o no hay mas files que puedan escribirse |
|
573 | 0 : Si no hay data o no hay mas files que puedan escribirse | |
| 567 | 1 : Si se escribio la data de un bloque en un file |
|
574 | 1 : Si se escribio la data de un bloque en un file | |
| 568 | """ |
|
575 | """ | |
| 569 | if self.dataOut.flagNoData: |
|
576 | if self.dataOut.flagNoData: | |
| 570 | return 0 |
|
577 | return 0 | |
| 571 |
|
578 | |||
| 572 | self.flagIsNewBlock = 0 |
|
579 | self.flagIsNewBlock = 0 | |
| 573 |
|
580 | |||
| 574 | if self.dataOut.flagDiscontinuousBlock: |
|
581 | if self.dataOut.flagDiscontinuousBlock: | |
| 575 | self.datablock.fill(0) |
|
582 | self.datablock.fill(0) | |
| 576 | self.profileIndex = 0 |
|
583 | self.profileIndex = 0 | |
| 577 | self.setNextFile() |
|
584 | self.setNextFile() | |
| 578 |
|
585 | |||
| 579 | if self.profileIndex == 0: |
|
586 | if self.profileIndex == 0: | |
| 580 | self.setBasicHeader() |
|
587 | self.setBasicHeader() | |
| 581 |
|
588 | |||
| 582 | self.datablock[:,self.profileIndex,:] = self.dataOut.data |
|
589 | self.datablock[:,self.profileIndex,:] = self.dataOut.data | |
| 583 |
|
590 | |||
| 584 | self.profileIndex += 1 |
|
591 | self.profileIndex += 1 | |
| 585 |
|
592 | |||
| 586 | if self.hasAllDataInBuffer(): |
|
593 | if self.hasAllDataInBuffer(): | |
| 587 | #if self.flagIsNewFile: |
|
594 | #if self.flagIsNewFile: | |
| 588 | self.writeNextBlock() |
|
595 | self.writeNextBlock() | |
| 589 | # self.setFirstHeader() |
|
596 | # self.setFirstHeader() | |
| 590 |
|
597 | |||
| 591 | return 1 |
|
598 | return 1 | |
| 592 |
|
599 | |||
| 593 | def __getBlockSize(self): |
|
600 | def __getBlockSize(self): | |
| 594 | ''' |
|
601 | ''' | |
| 595 | Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage |
|
602 | Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage | |
| 596 | ''' |
|
603 | ''' | |
| 597 |
|
604 | |||
| 598 | dtype_width = self.getDtypeWidth() |
|
605 | dtype_width = self.getDtypeWidth() | |
| 599 |
|
606 | |||
| 600 | blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2) |
|
607 | blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2) | |
| 601 |
|
608 | |||
| 602 | return blocksize |
|
609 | return blocksize | |
| 603 |
|
610 | |||
| 604 | def setFirstHeader(self): |
|
611 | def setFirstHeader(self): | |
| 605 |
|
612 | |||
| 606 | """ |
|
613 | """ | |
| 607 | Obtiene una copia del First Header |
|
614 | Obtiene una copia del First Header | |
| 608 |
|
615 | |||
| 609 | Affected: |
|
616 | Affected: | |
| 610 | self.systemHeaderObj |
|
617 | self.systemHeaderObj | |
| 611 | self.radarControllerHeaderObj |
|
618 | self.radarControllerHeaderObj | |
| 612 | self.dtype |
|
619 | self.dtype | |
| 613 |
|
620 | |||
| 614 | Return: |
|
621 | Return: | |
| 615 | None |
|
622 | None | |
| 616 | """ |
|
623 | """ | |
| 617 |
|
624 | |||
| 618 | self.systemHeaderObj = self.dataOut.systemHeaderObj.copy() |
|
625 | self.systemHeaderObj = self.dataOut.systemHeaderObj.copy() | |
| 619 | self.systemHeaderObj.nChannels = self.dataOut.nChannels |
|
626 | self.systemHeaderObj.nChannels = self.dataOut.nChannels | |
| 620 | self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy() |
|
627 | self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy() | |
| 621 |
|
628 | |||
| 622 | self.processingHeaderObj.dtype = 0 # Voltage |
|
629 | self.processingHeaderObj.dtype = 0 # Voltage | |
| 623 | self.processingHeaderObj.blockSize = self.__getBlockSize() |
|
630 | self.processingHeaderObj.blockSize = self.__getBlockSize() | |
| 624 | self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock |
|
631 | self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock | |
| 625 | self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile |
|
632 | self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile | |
| 626 | self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows |
|
633 | self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows | |
| 627 | self.processingHeaderObj.nCohInt = self.dataOut.nCohInt |
|
634 | self.processingHeaderObj.nCohInt = self.dataOut.nCohInt | |
| 628 | self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage |
|
635 | self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage | |
| 629 | self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage |
|
636 | self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage | |
| 630 |
|
637 | |||
| 631 | if self.dataOut.code is not None: |
|
638 | if self.dataOut.code is not None: | |
| 632 | self.processingHeaderObj.code = self.dataOut.code |
|
639 | self.processingHeaderObj.code = self.dataOut.code | |
| 633 | self.processingHeaderObj.nCode = self.dataOut.nCode |
|
640 | self.processingHeaderObj.nCode = self.dataOut.nCode | |
| 634 | self.processingHeaderObj.nBaud = self.dataOut.nBaud |
|
641 | self.processingHeaderObj.nBaud = self.dataOut.nBaud | |
| 635 |
|
642 | |||
| 636 | if self.processingHeaderObj.nWindows != 0: |
|
643 | if self.processingHeaderObj.nWindows != 0: | |
| 637 | self.processingHeaderObj.firstHeight = self.dataOut.heightList[0] |
|
644 | self.processingHeaderObj.firstHeight = self.dataOut.heightList[0] | |
| 638 | self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0] |
|
645 | self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0] | |
| 639 | self.processingHeaderObj.nHeights = self.dataOut.nHeights |
|
646 | self.processingHeaderObj.nHeights = self.dataOut.nHeights | |
| 640 | self.processingHeaderObj.samplesWin = self.dataOut.nHeights |
|
647 | self.processingHeaderObj.samplesWin = self.dataOut.nHeights | |
| 641 |
|
648 | |||
| 642 | self.processingHeaderObj.processFlags = self.getProcessFlags() |
|
649 | self.processingHeaderObj.processFlags = self.getProcessFlags() | |
| 643 |
|
650 | |||
| 644 | self.setBasicHeader() |
|
651 | self.setBasicHeader() | |
| @@ -1,1 +1,1 | |||||
| 1 | <Project description="HF_EXAMPLE" id="191" name="test01"><ReadUnit datatype="Spectra" id="1911" inputId="0" name="SpectraReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="datatype" value="SpectraReader" /><Parameter format="str" id="191112" name="path" value="/home/nanosat/data/sp1_f0" /><Parameter format="date" id="191113" name="startDate" value="2017/01/28" /><Parameter format="date" id="191114" name="endDate" value="2017/01/28" /><Parameter format="time" id="191115" name="startTime" value="00:00:00" /><Parameter format="time" id="191116" name="endTime" value="23:59:59" /><Parameter format="int" id="191118" name="cursor" value="5" /><Parameter format="int" id="191119" name="skip" value="177" /><Parameter format="int" id="191120" name="walk" value="1" /><Parameter format="int" id="191121" name="verbose" value="1" /><Parameter format="int" id="191122" name="online" value="0" /></Operation></ReadUnit><ProcUnit datatype="Parameters" id="1913" inputId="1911" name="ParametersProc"><Operation id="19131" name="run" priority="1" type="self" /><Operation id="19132" name="SpectralMoments" priority="2" type="other" /><Operation id="19133" name="PublishData" priority="3" type="other"><Parameter format="int" id="191331" name="zeromq" value="1" /><Parameter format="bool" id="191332" name="verbose" value="0" /><Parameter format="int" id="191333" name="delay" value="0" /></Operation></ProcUnit><ProcUnit datatype="Spectra" id="1912" inputId="1911" name="SpectraProc"><Operation id="19121" name="run" priority="1" type="self" /></ProcUnit></Project> |
|
1 | <Project description="test" id="191" name="john"><ReadUnit datatype="VoltageReader" id="1911" inputId="0" name="VoltageReader"><Operation id="19111" name="run" priority="1" type="self"><Parameter format="str" id="191111" name="server" value="john" /></Operation></ReadUnit><ProcUnit datatype="SpectraProc" id="1912" inputId="1911" name="SpectraProc"><Operation id="19121" name="run" priority="1" type="self"><Parameter format="int" id="191211" name="nFFTPoints" value="64" /></Operation><Operation id="19122" name="SpectraPlot" priority="2" type="other"><Parameter format="int" id="191221" name="id" value="500" /><Parameter format="str" id="191222" name="wintitle" value="Jicamarca Radio Observatory" /><Parameter format="int" id="191223" name="showprofile" value="0" /></Operation></ProcUnit></Project> No newline at end of file | |
General Comments 0
You need to be logged in to leave comments.
Login now