ascii_reader.py @ 6d7b5dd7

ESS_GUIESS_GUI_DocsESS_GUI_batch_fittingESS_GUI_bumps_abstractionESS_GUI_iss1116ESS_GUI_iss879ESS_GUI_iss959ESS_GUI_openclESS_GUI_orderingESS_GUI_sync_sascalccostrafo411magnetic_scattrelease-4.1.1release-4.1.2release-4.2.2release_4.0.1ticket-1009ticket-1094-headlessticket-1242-2d-resolutionticket-1243ticket-1249ticket885unittest-saveload

Last change on this file since 6d7b5dd7 was 86d13b5, checked in by ajj, 11 years ago
Update ascii reader to support semi-colon delimited files
Property mode set to `100644`
File size: 15.6 KB

Line
1	"""
2	ASCII reader
3	"""
4	############################################################################
5	#This software was developed by the University of Tennessee as part of the
6	#Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
7	#project funded by the US National Science Foundation.
8	#If you use DANSE applications to do scientific research that leads to
9	#publication, we ask that you acknowledge the use of the software with the
10	#following sentence:
11	#This work benefited from DANSE software developed under NSF award DMR-0520547.
12	#copyright 2008, University of Tennessee
13	#############################################################################
14
15
16	import numpy
17	import os
18	from sans.dataloader.data_info import Data1D
19
20	# Check whether we have a converter available
21	has_converter = True
22	try:
23	from sans.data_util.nxsunit import Converter
24	except:
25	has_converter = False
26	_ZERO = 1e-16
27
28
29	class Reader:
30	"""
31	Class to load ascii files (2, 3 or 4 columns).
32	"""
33	## File type
34	type_name = "ASCII"
35
36	## Wildcards
37	type = ["ASCII files (.txt)\|.txt",
38	"ASCII files (.dat)\|.dat",
39	"ASCII files (.abs)\|.abs",
40	"CSV files (.csv)\|.csv"]
41	## List of allowed extensions
42	ext = ['.txt', '.TXT', '.dat', '.DAT', '.abs', '.ABS', 'csv', 'CSV']
43
44	## Flag to bypass extension check
45	allow_all = True
46
47	def read(self, path):
48	"""
49	Load data file
50
51	:param path: file path
52
53	:return: Data1D object, or None
54
55	:raise RuntimeError: when the file can't be opened
56	:raise ValueError: when the length of the data vectors are inconsistent
57	"""
58	if os.path.isfile(path):
59	basename = os.path.basename(path)
60	_, extension = os.path.splitext(basename)
61	if self.allow_all or extension.lower() in self.ext:
62	try:
63	# Read in binary mode since GRASP frequently has no-ascii
64	# characters that brakes the open operation
65	input_f = open(path,'rb')
66	except:
67	raise RuntimeError, "ascii_reader: cannot open %s" % path
68	buff = input_f.read()
69	lines = buff.split('\n')
70
71	#Jae could not find python universal line spliter:
72	#keep the below for now
73	# some ascii data has \r line separator,
74	# try it when the data is on only one long line
75	if len(lines) < 2 :
76	lines = buff.split('\r')
77
78	x = numpy.zeros(0)
79	y = numpy.zeros(0)
80	dy = numpy.zeros(0)
81	dx = numpy.zeros(0)
82
83	#temp. space to sort data
84	tx = numpy.zeros(0)
85	ty = numpy.zeros(0)
86	tdy = numpy.zeros(0)
87	tdx = numpy.zeros(0)
88
89	output = Data1D(x, y, dy=dy, dx=dx)
90	self.filename = output.filename = basename
91
92	data_conv_q = None
93	data_conv_i = None
94
95	if has_converter == True and output.x_unit != '1/A':
96	data_conv_q = Converter('1/A')
97	# Test it
98	data_conv_q(1.0, output.x_unit)
99
100	if has_converter == True and output.y_unit != '1/cm':
101	data_conv_i = Converter('1/cm')
102	# Test it
103	data_conv_i(1.0, output.y_unit)
104
105	# The first good line of data will define whether
106	# we have 2-column or 3-column ascii
107	has_error_dx = None
108	has_error_dy = None
109
110	#Initialize counters for data lines and header lines.
111	is_data = False # Has more than 5 lines
112	# More than "5" lines of data is considered as actual
113	# data unless that is the only data
114	mum_data_lines = 5
115	# To count # of current data candidate lines
116	i = -1
117	# To count total # of previous data candidate lines
118	i1 = -1
119	# To count # of header lines
120	j = -1
121	# Helps to count # of header lines
122	j1 = -1
123	#minimum required number of columns of data; ( <= 4).
124	lentoks = 2
125	for line in lines:
126	# Initial try for CSV (split on ,)
127	toks = line.split(',')
128	# Now try SCSV (split on ;)
129	if len(toks) < 2:
130	toks = line.split(';')
131	# Now go for whitespace
132	if len(toks) < 2:
133	toks = line.split()
134	try:
135	#Make sure that all columns are numbers.
136	for colnum in range(len(toks)):
137	float(toks[colnum])
138
139	_x = float(toks[0])
140	_y = float(toks[1])
141
142	#Reset the header line counters
143	if j == j1:
144	j = 0
145	j1 = 0
146
147	if i > 1:
148	is_data = True
149
150	if data_conv_q is not None:
151	_x = data_conv_q(_x, units=output.x_unit)
152
153	if data_conv_i is not None:
154	_y = data_conv_i(_y, units=output.y_unit)
155
156	# If we have an extra token, check
157	# whether it can be interpreted as a
158	# third column.
159	_dy = None
160	if len(toks) > 2:
161	try:
162	_dy = float(toks[2])
163
164	if data_conv_i is not None:
165	_dy = data_conv_i(_dy, units=output.y_unit)
166
167	except:
168	# The third column is not a float, skip it.
169	pass
170
171	# If we haven't set the 3rd column
172	# flag, set it now.
173	if has_error_dy == None:
174	has_error_dy = False if _dy == None else True
175
176	#Check for dx
177	_dx = None
178	if len(toks) > 3:
179	try:
180	_dx = float(toks[3])
181
182	if data_conv_i is not None:
183	_dx = data_conv_i(_dx, units=output.x_unit)
184
185	except:
186	# The 4th column is not a float, skip it.
187	pass
188
189	# If we haven't set the 3rd column
190	# flag, set it now.
191	if has_error_dx == None:
192	has_error_dx = False if _dx == None else True
193
194	#After talked with PB, we decided to take care of only
195	# 4 columns of data for now.
196	#number of columns in the current line
197	#To remember the # of columns in the current
198	#line of data
199	new_lentoks = len(toks)
200
201	#If the previous columns not equal to the current,
202	#mark the previous as non-data and reset the dependents.
203	if lentoks != new_lentoks:
204	if is_data == True:
205	break
206	else:
207	i = -1
208	i1 = 0
209	j = -1
210	j1 = -1
211
212	#Delete the previously stored lines of data candidates
213	# if is not data.
214	if i < 0 and -1 < i1 < mum_data_lines and \
215	is_data == False:
216	try:
217	x = numpy.zeros(0)
218	y = numpy.zeros(0)
219	except:
220	pass
221
222	x = numpy.append(x, _x)
223	y = numpy.append(y, _y)
224
225	if has_error_dy == True:
226	#Delete the previously stored lines of
227	# data candidates if is not data.
228	if i < 0 and -1 < i1 < mum_data_lines and \
229	is_data == False:
230	try:
231	dy = numpy.zeros(0)
232	except:
233	pass
234	dy = numpy.append(dy, _dy)
235
236	if has_error_dx == True:
237	#Delete the previously stored lines of
238	# data candidates if is not data.
239	if i < 0 and -1 < i1 < mum_data_lines and \
240	is_data == False:
241	try:
242	dx = numpy.zeros(0)
243	except:
244	pass
245	dx = numpy.append(dx, _dx)
246
247	#Same for temp.
248	#Delete the previously stored lines of data candidates
249	# if is not data.
250	if i < 0 and -1 < i1 < mum_data_lines and\
251	is_data == False:
252	try:
253	tx = numpy.zeros(0)
254	ty = numpy.zeros(0)
255	except:
256	pass
257
258	tx = numpy.append(tx, _x)
259	ty = numpy.append(ty, _y)
260
261	if has_error_dy == True:
262	#Delete the previously stored lines of
263	# data candidates if is not data.
264	if i < 0 and -1 < i1 < mum_data_lines and \
265	is_data == False:
266	try:
267	tdy = numpy.zeros(0)
268	except:
269	pass
270	tdy = numpy.append(tdy, _dy)
271	if has_error_dx == True:
272	#Delete the previously stored lines of
273	# data candidates if is not data.
274	if i < 0 and -1 < i1 < mum_data_lines and \
275	is_data == False:
276	try:
277	tdx = numpy.zeros(0)
278	except:
279	pass
280	tdx = numpy.append(tdx, _dx)
281
282	#reset i1 and flag lentoks for the next
283	if lentoks < new_lentoks:
284	if is_data == False:
285	i1 = -1
286	#To remember the # of columns on the current line
287	# for the next line of data
288	lentoks = len(toks)
289
290	#Reset # of header lines and counts #
291	# of data candidate lines
292	if j == 0 and j1 == 0:
293	i1 = i + 1
294	i += 1
295	except:
296
297	# It is data and meet non - number, then stop reading
298	if is_data == True:
299	break
300	lentoks = 2
301	#Counting # of header lines
302	j += 1
303	if j == j1 + 1:
304	j1 = j
305	else:
306	j = -1
307	#Reset # of lines of data candidates
308	i = -1
309
310	# Couldn't parse this line, skip it
311	pass
312
313	input_f.close()
314	# Sanity check
315	if has_error_dy == True and not len(y) == len(dy):
316	msg = "ascii_reader: y and dy have different length"
317	raise RuntimeError, msg
318	if has_error_dx == True and not len(x) == len(dx):
319	msg = "ascii_reader: y and dy have different length"
320	raise RuntimeError, msg
321	# If the data length is zero, consider this as
322	# though we were not able to read the file.
323	if len(x) == 0:
324	raise RuntimeError, "ascii_reader: could not load file"
325
326	#Let's re-order the data to make cal.
327	# curve look better some cases
328	ind = numpy.lexsort((ty, tx))
329	for i in ind:
330	x[i] = tx[ind[i]]
331	y[i] = ty[ind[i]]
332	if has_error_dy == True:
333	dy[i] = tdy[ind[i]]
334	if has_error_dx == True:
335	dx[i] = tdx[ind[i]]
336	# Zeros in dx, dy
337	if has_error_dx:
338	dx[dx == 0] = _ZERO
339	if has_error_dy:
340	dy[dy == 0] = _ZERO
341	#Data
342	output.x = x[x != 0]
343	output.y = y[x != 0]
344	output.dy = dy[x != 0] if has_error_dy == True\
345	else numpy.zeros(len(output.y))
346	output.dx = dx[x != 0] if has_error_dx == True\
347	else numpy.zeros(len(output.x))
348
349	if data_conv_q is not None:
350	output.xaxis("\\rm{Q}", output.x_unit)
351	else:
352	output.xaxis("\\rm{Q}", 'A^{-1}')
353	if data_conv_i is not None:
354	output.yaxis("\\rm{Intensity}", output.y_unit)
355	else:
356	output.yaxis("\\rm{Intensity}", "cm^{-1}")
357
358	# Store loading process information
359	output.meta_data['loader'] = self.type_name
360	if len(output.x) < 1:
361	raise RuntimeError, "%s is empty" % path
362	return output
363
364	else:
365	raise RuntimeError, "%s is not a file" % path
366	return None

Note: See TracBrowser for help on using the repository browser.

SasView

source: sasview/src/sans/dataloader/readers/ascii_reader.py @ 6d7b5dd7

Download in other formats: