ascii_reader.py @ a24f530

ESS_GUIESS_GUI_DocsESS_GUI_batch_fittingESS_GUI_bumps_abstractionESS_GUI_iss1116ESS_GUI_iss879ESS_GUI_iss959ESS_GUI_openclESS_GUI_orderingESS_GUI_sync_sascalccostrafo411magnetic_scattrelease-4.1.1release-4.1.2release-4.2.2release_4.0.1ticket-1009ticket-1094-headlessticket-1242-2d-resolutionticket-1243ticket-1249ticket885unittest-saveload

Last change on this file since a24f530 was ef9d209, checked in by Jae Cho <jhjcho@…>, 13 years ago
added csv format: but read floats only but not strings
Property mode set to `100644`
File size: 16.0 KB

Rev	Line
[8bd8ea4]	1
	2
[0997158f]	3	############################################################################
	4	#This software was developed by the University of Tennessee as part of the
	5	#Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
	6	#project funded by the US National Science Foundation.
	7	#If you use DANSE applications to do scientific research that leads to
	8	#publication, we ask that you acknowledge the use of the software with the
	9	#following sentence:
	10	#This work benefited from DANSE software developed under NSF award DMR-0520547.
	11	#copyright 2008, University of Tennessee
	12	#############################################################################
	13
[8bd8ea4]	14
	15	import numpy
	16	import os
[ad8034f]	17	from sans.dataloader.data_info import Data1D
[8bd8ea4]	18
[daa56d0]	19	# Check whether we have a converter available
[99d1af6]	20	has_converter = True
	21	try:
	22	from data_util.nxsunit import Converter
	23	except:
	24	has_converter = False
[da96629]	25	_ZERO = 1e-16
[99d1af6]	26
[8bd8ea4]	27	class Reader:
	28	"""
[0997158f]	29	Class to load ascii files (2, 3 or 4 columns).
[8bd8ea4]	30	"""
[8780e9a]	31	## File type
[28caa03]	32	type_name = "ASCII"
	33
	34	## Wildcards
[8780e9a]	35	type = ["ASCII files (.txt)\|.txt",
[470bf7e]	36	"ASCII files (.dat)\|.dat",
[ef9d209]	37	"ASCII files (.abs)\|.abs",
	38	"CSV files (.csv)\|.csv"]
[8bd8ea4]	39	## List of allowed extensions
[ef9d209]	40	ext = ['.txt', '.TXT', '.dat', '.DAT', '.abs', '.ABS', 'csv', 'CSV']
[8bd8ea4]	41
[e082e2c]	42	## Flag to bypass extension check
	43	allow_all = True
	44
[8bd8ea4]	45	def read(self, path):
	46	"""
[0997158f]	47	Load data file
	48
	49	:param path: file path
	50
	51	:return: Data1D object, or None
	52
	53	:raise RuntimeError: when the file can't be opened
	54	:raise ValueError: when the length of the data vectors are inconsistent
[8bd8ea4]	55	"""
	56	if os.path.isfile(path):
	57	basename = os.path.basename(path)
[a7a5886]	58	_, extension = os.path.splitext(basename)
[e082e2c]	59	if self.allow_all or extension.lower() in self.ext:
[8bd8ea4]	60	try:
	61	input_f = open(path,'r')
	62	except :
	63	raise RuntimeError, "ascii_reader: cannot open %s" % path
	64	buff = input_f.read()
	65	lines = buff.split('\n')
[470bf7e]	66
[a7a5886]	67	#Jae could not find python universal line spliter:
	68	#keep the below for now
	69	# some ascii data has \r line separator,
	70	# try it when the data is on only one long line
[470bf7e]	71	if len(lines) < 2 :
	72	lines = buff.split('\r')
	73
[8bd8ea4]	74	x = numpy.zeros(0)
	75	y = numpy.zeros(0)
	76	dy = numpy.zeros(0)
[de1da34]	77	dx = numpy.zeros(0)
	78
	79	#temp. space to sort data
	80	tx = numpy.zeros(0)
	81	ty = numpy.zeros(0)
	82	tdy = numpy.zeros(0)
	83	tdx = numpy.zeros(0)
	84
	85	output = Data1D(x, y, dy=dy, dx=dx)
[8bd8ea4]	86	self.filename = output.filename = basename
[99d1af6]	87
	88	data_conv_q = None
	89	data_conv_i = None
	90
[ca10d8e]	91	if has_converter == True and output.x_unit != '1/A':
	92	data_conv_q = Converter('1/A')
[99d1af6]	93	# Test it
	94	data_conv_q(1.0, output.x_unit)
	95
[ca10d8e]	96	if has_converter == True and output.y_unit != '1/cm':
	97	data_conv_i = Converter('1/cm')
[99d1af6]	98	# Test it
	99	data_conv_i(1.0, output.y_unit)
	100
[8bd8ea4]	101
	102	# The first good line of data will define whether
	103	# we have 2-column or 3-column ascii
[de1da34]	104	has_error_dx = None
	105	has_error_dy = None
[8bd8ea4]	106
[892f246]	107	#Initialize counters for data lines and header lines.
[0e5e586]	108	is_data = False #Has more than 5 lines
[a7a5886]	109	# More than "5" lines of data is considered as actual
	110	# data unless that is the only data
	111	mum_data_lines = 5
	112	# To count # of current data candidate lines
	113	i = -1
	114	# To count total # of previous data candidate lines
	115	i1 = -1
	116	# To count # of header lines
	117	j = -1
	118	# Helps to count # of header lines
	119	j1 = -1
	120	#minimum required number of columns of data; ( <= 4).
	121	lentoks = 2
[8bd8ea4]	122	for line in lines:
[ef9d209]	123	toks = line.split(',')
	124	if len(toks) < 2:
	125	toks = line.split()
[8bd8ea4]	126	try:
[5f2d3c78]	127	#Make sure that all columns are numbers.
	128	for colnum in range(len(toks)):
	129	float(toks[colnum])
	130
[8bd8ea4]	131	_x = float(toks[0])
	132	_y = float(toks[1])
	133
[892f246]	134	#Reset the header line counters
	135	if j == j1:
	136	j = 0
	137	j1 = 0
	138
	139	if i > 1:
	140	is_data = True
[d508be9]	141
[99d1af6]	142	if data_conv_q is not None:
	143	_x = data_conv_q(_x, units=output.x_unit)
	144
	145	if data_conv_i is not None:
	146	_y = data_conv_i(_y, units=output.y_unit)
	147
[8bd8ea4]	148	# If we have an extra token, check
	149	# whether it can be interpreted as a
	150	# third column.
	151	_dy = None
[a7a5886]	152	if len(toks) > 2:
[8bd8ea4]	153	try:
	154	_dy = float(toks[2])
[99d1af6]	155
	156	if data_conv_i is not None:
	157	_dy = data_conv_i(_dy, units=output.y_unit)
	158
[8bd8ea4]	159	except:
	160	# The third column is not a float, skip it.
	161	pass
	162
	163	# If we haven't set the 3rd column
	164	# flag, set it now.
[de1da34]	165	if has_error_dy == None:
	166	has_error_dy = False if _dy == None else True
	167
	168	#Check for dx
	169	_dx = None
[a7a5886]	170	if len(toks) > 3:
[de1da34]	171	try:
	172	_dx = float(toks[3])
	173
	174	if data_conv_i is not None:
	175	_dx = data_conv_i(_dx, units=output.x_unit)
	176
	177	except:
	178	# The 4th column is not a float, skip it.
	179	pass
	180
	181	# If we haven't set the 3rd column
	182	# flag, set it now.
	183	if has_error_dx == None:
	184	has_error_dx = False if _dx == None else True
[892f246]	185
[a7a5886]	186	#After talked with PB, we decided to take care of only
	187	# 4 columns of data for now.
[d508be9]	188	#number of columns in the current line
[a7a5886]	189	#To remember the # of columns in the current
	190	#line of data
[0e5e586]	191	new_lentoks = len(toks)
[d508be9]	192
[a7a5886]	193	#If the previous columns not equal to the current,
	194	#mark the previous as non-data and reset the dependents.
[272b107]	195	if lentoks != new_lentoks :
	196	if is_data == True:
	197	break
	198	else:
[d508be9]	199	i = -1
	200	i1 = 0
	201	j = -1
	202	j1 = -1
	203
[272b107]	204
[a7a5886]	205	#Delete the previously stored lines of data candidates
	206	# if is not data.
	207	if i < 0 and -1 < i1 < mum_data_lines and \
	208	is_data == False:
[892f246]	209	try:
[a7a5886]	210	x = numpy.zeros(0)
	211	y = numpy.zeros(0)
[892f246]	212	except:
	213	pass
	214
[8bd8ea4]	215	x = numpy.append(x, _x)
	216	y = numpy.append(y, _y)
[892f246]	217
[de1da34]	218	if has_error_dy == True:
[a7a5886]	219	#Delete the previously stored lines of
	220	# data candidates if is not data.
	221	if i < 0 and -1 < i1 < mum_data_lines and \
	222	is_data == False:
[892f246]	223	try:
	224	dy = numpy.zeros(0)
	225	except:
	226	pass
[8bd8ea4]	227	dy = numpy.append(dy, _dy)
[892f246]	228
[de1da34]	229	if has_error_dx == True:
[a7a5886]	230	#Delete the previously stored lines of
	231	# data candidates if is not data.
	232	if i < 0 and -1 < i1 < mum_data_lines and \
	233	is_data == False:
[892f246]	234	try:
	235	dx = numpy.zeros(0)
	236	except:
	237	pass
[de1da34]	238	dx = numpy.append(dx, _dx)
	239
	240	#Same for temp.
[a7a5886]	241	#Delete the previously stored lines of data candidates
	242	# if is not data.
	243	if i < 0 and -1 < i1 < mum_data_lines and\
	244	is_data == False:
[892f246]	245	try:
	246	tx = numpy.zeros(0)
	247	ty = numpy.zeros(0)
	248	except:
[a7a5886]	249	pass
[892f246]	250
[de1da34]	251	tx = numpy.append(tx, _x)
	252	ty = numpy.append(ty, _y)
[892f246]	253
[de1da34]	254	if has_error_dy == True:
[a7a5886]	255	#Delete the previously stored lines of
	256	# data candidates if is not data.
	257	if i < 0 and -1 < i1 < mum_data_lines and \
	258	is_data == False:
[892f246]	259	try:
	260	tdy = numpy.zeros(0)
	261	except:
	262	pass
[de1da34]	263	tdy = numpy.append(tdy, _dy)
	264	if has_error_dx == True:
[a7a5886]	265	#Delete the previously stored lines of
	266	# data candidates if is not data.
	267	if i < 0 and -1 < i1 < mum_data_lines and \
	268	is_data == False:
[892f246]	269	try:
	270	tdx = numpy.zeros(0)
	271	except:
[a7a5886]	272	pass
[de1da34]	273	tdx = numpy.append(tdx, _dx)
[d508be9]	274
	275	#reset i1 and flag lentoks for the next
[a7a5886]	276	if lentoks < new_lentoks:
[d508be9]	277	if is_data == False:
	278	i1 = -1
[a7a5886]	279	#To remember the # of columns on the current line
	280	# for the next line of data
[0e5e586]	281	lentoks = len(toks)
[8bd8ea4]	282
[a7a5886]	283	#Reset # of header lines and counts #
	284	# of data candidate lines
	285	if j == 0 and j1 == 0:
[892f246]	286	i1 = i + 1
[a7a5886]	287	i += 1
[8bd8ea4]	288	except:
[892f246]	289
	290	# It is data and meet non - number, then stop reading
	291	if is_data == True:
	292	break
[d508be9]	293	lentoks = 2
[892f246]	294	#Counting # of header lines
[a7a5886]	295	j += 1
	296	if j == j1 + 1:
[892f246]	297	j1 = j
	298	else:
	299	j = -1
	300	#Reset # of lines of data candidates
	301	i = -1
	302
[8bd8ea4]	303	# Couldn't parse this line, skip it
	304	pass
[892f246]	305
[c7c5ef8]	306	input_f.close()
[8bd8ea4]	307	# Sanity check
[de1da34]	308	if has_error_dy == True and not len(y) == len(dy):
[a7a5886]	309	msg = "ascii_reader: y and dy have different length"
	310	raise RuntimeError, msg
[de1da34]	311	if has_error_dx == True and not len(x) == len(dx):
[a7a5886]	312	msg = "ascii_reader: y and dy have different length"
	313	raise RuntimeError, msg
[8bd8ea4]	314	# If the data length is zero, consider this as
	315	# though we were not able to read the file.
[a7a5886]	316	if len(x) == 0:
[daa56d0]	317	raise RuntimeError, "ascii_reader: could not load file"
[de1da34]	318
[a7a5886]	319	#Let's re-order the data to make cal.
	320	# curve look better some cases
	321	ind = numpy.lexsort((ty, tx))
[de1da34]	322	for i in ind:
	323	x[i] = tx[ind[i]]
	324	y[i] = ty[ind[i]]
	325	if has_error_dy == True:
	326	dy[i] = tdy[ind[i]]
	327	if has_error_dx == True:
	328	dx[i] = tdx[ind[i]]
[da96629]	329	# Zeros in dx, dy
	330	if has_error_dx:
	331	dx[dx==0] = _ZERO
	332	if has_error_dy:
	333	dy[dy==0] = _ZERO
[892f246]	334	#Data
[121c224]	335	output.x = x[x!=0]
	336	output.y = y[x!=0]
[5f03524]	337	output.dy = dy[x!=0] if has_error_dy == True else numpy.zeros(len(output.y))
	338	output.dx = dx[x!=0] if has_error_dx == True else numpy.zeros(len(output.x))
[fca90f82]	339
[99d1af6]	340	if data_conv_q is not None:
	341	output.xaxis("\\rm{Q}", output.x_unit)
	342	else:
	343	output.xaxis("\\rm{Q}", 'A^{-1}')
	344	if data_conv_i is not None:
[0e2aa40]	345	output.yaxis("\\rm{Intensity}", output.y_unit)
[99d1af6]	346	else:
[0e2aa40]	347	output.yaxis("\\rm{Intensity}","cm^{-1}")
[fe78c7b]	348
	349	# Store loading process information
	350	output.meta_data['loader'] = self.type_name
	351
[8bd8ea4]	352	return output
[892f246]	353
[8bd8ea4]	354	else:
	355	raise RuntimeError, "%s is not a file" % path
	356	return None
	357
	358	if __name__ == "__main__":
	359	reader = Reader()
	360	#print reader.read("../test/test_3_columns.txt")
	361	print reader.read("../test/empty.txt")
	362
	363
	364

Note: See TracBrowser for help on using the repository browser.

SasView

source: sasview/sansdataloader/src/sans/dataloader/readers/ascii_reader.py @ a24f530

Download in other formats: