loader.py @ 0e0c645

ticket-1243

Last change on this file since 0e0c645 was b1ec23d, checked in by krzywon, 6 years ago
Remove unneeded shape check, give better error when saving fails, and data info cleanup.
Property mode set to `100644`
File size: 15.7 KB

Line
1	"""
2	File handler to support different file extensions.
3	Uses reflectometer registry utility.
4
5	The default readers are found in the 'readers' sub-module
6	and registered by default at initialization time.
7
8	To add a new default reader, one must register it in
9	the register_readers method found in readers/__init__.py.
10
11	A utility method (find_plugins) is available to inspect
12	a directory (for instance, a user plug-in directory) and
13	look for new readers/writers.
14	"""
15	#####################################################################
16	# This software was developed by the University of Tennessee as part of the
17	# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
18	# project funded by the US National Science Foundation.
19	# See the license text in license.txt
20	# copyright 2008, University of Tennessee
21	######################################################################
22
23	import os
24	import sys
25	import logging
26	import time
27	from zipfile import ZipFile
28
29	from sas.sascalc.data_util.registry import ExtensionRegistry
30
31	# Default readers are defined in the readers sub-module
32	from . import readers
33	from .loader_exceptions import NoKnownLoaderException, FileContentsException,\
34	DefaultReaderException
35	from .readers import ascii_reader
36	from .readers import cansas_reader
37	from .readers import cansas_reader_HDF5
38
39	logger = logging.getLogger(__name__)
40
41
42	class Registry(ExtensionRegistry):
43	"""
44	Registry class for file format extensions.
45	Readers and writers are supported.
46	"""
47	def __init__(self):
48	super(Registry, self).__init__()
49
50	# Writers
51	self.writers = {}
52
53	# List of wildcards
54	self.wildcards = ['All (.)\|.']
55
56	# Creation time, for testing
57	self._created = time.time()
58
59	# Register default readers
60	readers.read_associations(self)
61
62	def load(self, path, format=None):
63	"""
64	Call the loader for the file type of path.
65
66	:param path: file path
67	:param format: explicit extension, to force the use
68	of a particular reader
69
70	Defaults to the ascii (multi-column), cansas XML, and cansas NeXuS
71	readers if no reader was registered for the file's extension.
72	"""
73	# Gets set to a string if the file has an associated reader that fails
74	msg_from_reader = None
75	try:
76	return super(Registry, self).load(path, format=format)
77	#except Exception: raise # for debugging, don't use fallback loader
78	except NoKnownLoaderException as nkl_e:
79	pass # Try the ASCII reader
80	except FileContentsException as fc_exc:
81	# File has an associated reader but it failed.
82	# Save the error message to display later, but try the 3 default loaders
83	msg_from_reader = fc_exc.message
84	except Exception:
85	pass
86
87	# File has no associated reader, or the associated reader failed.
88	# Try the ASCII reader
89	try:
90	ascii_loader = ascii_reader.Reader()
91	return ascii_loader.read(path)
92	except NoKnownLoaderException:
93	pass # Try the Cansas XML reader
94	except DefaultReaderException:
95	pass # Loader specific error to try the cansas XML reader
96	except FileContentsException as e:
97	if msg_from_reader is None:
98	raise RuntimeError(e.message)
99
100	# ASCII reader failed - try CanSAS xML reader
101	try:
102	cansas_loader = cansas_reader.Reader()
103	return cansas_loader.read(path)
104	except NoKnownLoaderException:
105	pass # Try the NXcanSAS reader
106	except DefaultReaderException:
107	pass # Loader specific error to try the NXcanSAS reader
108	except FileContentsException as e:
109	if msg_from_reader is None:
110	raise RuntimeError(e.message)
111	except Exception:
112	pass
113
114	# CanSAS XML reader failed - try NXcanSAS reader
115	try:
116	cansas_nexus_loader = cansas_reader_HDF5.Reader()
117	return cansas_nexus_loader.read(path)
118	except DefaultReaderException as e:
119	logging.error("No default loader can load the data")
120	# No known reader available. Give up and throw an error
121	if msg_from_reader is None:
122	msg = "\nUnknown data format: {}.\nThe file is not a ".format(path)
123	msg += "known format that can be loaded by SasView.\n"
124	raise NoKnownLoaderException(msg)
125	else:
126	# Associated reader and default readers all failed.
127	# Show error message from associated reader
128	raise RuntimeError(msg_from_reader)
129	except FileContentsException as e:
130	err_msg = msg_from_reader if msg_from_reader is not None else e.message
131	raise RuntimeError(err_msg)
132
133	def find_plugins(self, dir):
134	"""
135	Find readers in a given directory. This method
136	can be used to inspect user plug-in directories to
137	find new readers/writers.
138
139	:param dir: directory to search into
140	:return: number of readers found
141	"""
142	readers_found = 0
143	temp_path = os.path.abspath(dir)
144	if not os.path.isdir(temp_path):
145	temp_path = os.path.join(os.getcwd(), dir)
146	if not os.path.isdir(temp_path):
147	temp_path = os.path.join(os.path.dirname(__file__), dir)
148	if not os.path.isdir(temp_path):
149	temp_path = os.path.join(os.path.dirname(sys.path[0]), dir)
150
151	dir = temp_path
152	# Check whether the directory exists
153	if not os.path.isdir(dir):
154	msg = "DataLoader couldn't locate DataLoader plugin folder."
155	msg += """ "%s" does not exist""" % dir
156	logger.warning(msg)
157	return readers_found
158
159	for item in os.listdir(dir):
160	full_path = os.path.join(dir, item)
161	if os.path.isfile(full_path):
162
163	# Process python files
164	if item.endswith('.py'):
165	toks = os.path.splitext(os.path.basename(item))
166	try:
167	sys.path.insert(0, os.path.abspath(dir))
168	module = __import__(toks[0], globals(), locals())
169	if self._identify_plugin(module):
170	readers_found += 1
171	except:
172	msg = "Loader: Error importing "
173	msg += "%s\n %s" % (item, sys.exc_value)
174	logger.error(msg)
175
176	# Process zip files
177	elif item.endswith('.zip'):
178	try:
179	# Find the modules in the zip file
180	zfile = ZipFile(item)
181	nlist = zfile.namelist()
182
183	sys.path.insert(0, item)
184	for mfile in nlist:
185	try:
186	# Change OS path to python path
187	fullname = mfile.replace('/', '.')
188	fullname = os.path.splitext(fullname)[0]
189	module = __import__(fullname, globals(),
190	locals(), [""])
191	if self._identify_plugin(module):
192	readers_found += 1
193	except:
194	msg = "Loader: Error importing"
195	msg += " %s\n %s" % (mfile, sys.exc_value)
196	logger.error(msg)
197
198	except:
199	msg = "Loader: Error importing "
200	msg += " %s\n %s" % (item, sys.exc_value)
201	logger.error(msg)
202
203	return readers_found
204
205	def associate_file_type(self, ext, module):
206	"""
207	Look into a module to find whether it contains a
208	Reader class. If so, APPEND it to readers and (potentially)
209	to the list of writers for the given extension
210
211	:param ext: file extension [string]
212	:param module: module object
213	"""
214	reader_found = False
215
216	if hasattr(module, "Reader"):
217	try:
218	# Find supported extensions
219	loader = module.Reader()
220	if ext not in self.loaders:
221	self.loaders[ext] = []
222	# Append the new reader to the list
223	self.loaders[ext].append(loader.read)
224
225	reader_found = True
226
227	# Keep track of wildcards
228	type_name = module.__name__
229	if hasattr(loader, 'type_name'):
230	type_name = loader.type_name
231
232	wcard = "%s files (%s)\|%s" % (type_name, ext.lower(),
233	ext.lower())
234	if wcard not in self.wildcards:
235	self.wildcards.append(wcard)
236
237	# Check whether writing is supported
238	if hasattr(loader, 'write'):
239	if ext not in self.writers:
240	self.writers[ext] = []
241	# Append the new writer to the list
242	self.writers[ext].append(loader.write)
243
244	except:
245	msg = "Loader: Error accessing"
246	msg += " Reader in %s\n %s" % (module.__name__, sys.exc_value)
247	logger.error(msg)
248	return reader_found
249
250	def associate_file_reader(self, ext, loader):
251	"""
252	Append a reader object to readers
253
254	:param ext: file extension [string]
255	:param module: reader object
256	"""
257	reader_found = False
258
259	try:
260	# Find supported extensions
261	if ext not in self.loaders:
262	self.loaders[ext] = []
263	# Append the new reader to the list
264	self.loaders[ext].append(loader.read)
265
266	reader_found = True
267
268	# Keep track of wildcards
269	if hasattr(loader, 'type_name'):
270	type_name = loader.type_name
271
272	wcard = "%s files (%s)\|%s" % (type_name, ext.lower(),
273	ext.lower())
274	if wcard not in self.wildcards:
275	self.wildcards.append(wcard)
276
277	except:
278	msg = "Loader: Error accessing Reader "
279	msg += "in %s\n %s" % (loader.__name__, sys.exc_value)
280	logger.error(msg)
281	return reader_found
282
283	def _identify_plugin(self, module):
284	"""
285	Look into a module to find whether it contains a
286	Reader class. If so, add it to readers and (potentially)
287	to the list of writers.
288	:param module: module object
289
290	"""
291	reader_found = False
292
293	if hasattr(module, "Reader"):
294	try:
295	# Find supported extensions
296	loader = module.Reader()
297	for ext in loader.ext:
298	if ext not in self.loaders:
299	self.loaders[ext] = []
300	# When finding a reader at run time,
301	# treat this reader as the new default
302	self.loaders[ext].insert(0, loader.read)
303
304	reader_found = True
305
306	# Keep track of wildcards
307	type_name = module.__name__
308	if hasattr(loader, 'type_name'):
309	type_name = loader.type_name
310	wcard = "%s files (%s)\|%s" % (type_name, ext.lower(),
311	ext.lower())
312	if wcard not in self.wildcards:
313	self.wildcards.append(wcard)
314
315	# Check whether writing is supported
316	if hasattr(loader, 'write'):
317	for ext in loader.ext:
318	if ext not in self.writers:
319	self.writers[ext] = []
320	self.writers[ext].insert(0, loader.write)
321
322	except:
323	msg = "Loader: Error accessing Reader"
324	msg += " in %s\n %s" % (module.__name__, sys.exc_value)
325	logger.error(msg)
326	return reader_found
327
328	def lookup_writers(self, path):
329	"""
330	:return: the loader associated with the file type of path.
331	:Raises ValueError: if file type is not known.
332	"""
333	# Find matching extensions
334	extlist = [ext for ext in self.extensions() if path.endswith(ext)]
335	# Sort matching extensions by decreasing order of length
336	extlist.sort(key=len)
337	# Combine loaders for matching extensions into one big list
338	writers = []
339	for L in [self.writers[ext] for ext in extlist]:
340	writers.extend(L)
341	# Remove duplicates if they exist
342	if len(writers) != len(set(writers)):
343	result = []
344	for L in writers:
345	if L not in result:
346	result.append(L)
347	writers = L
348	# Raise an error if there are no matching extensions
349	if len(writers) == 0:
350	raise ValueError("Unknown file type for " + path)
351	# All done
352	return writers
353
354	def save(self, path, data, format=None):
355	"""
356	Call the writer for the file type of path.
357
358	Raises ValueError if no writer is available.
359	Raises KeyError if format is not available.
360	May raise a writer-defined exception if writer fails.
361	"""
362	if format is None:
363	writers = self.lookup_writers(path)
364	else:
365	writers = self.writers[format]
366	for fn in writers:
367	try:
368	return fn(path, data)
369	except Exception as exc:
370	msg = "Saving file {} using the {} writer failed.\n".format(
371	path, type(fn).__name__)
372	msg += str(exc)
373	logger.exception(msg) # give other loaders a chance to succeed
374
375
376	class Loader(object):
377	"""
378	Utility class to use the Registry as a singleton.
379	"""
380	## Registry instance
381	__registry = Registry()
382
383	def associate_file_type(self, ext, module):
384	"""
385	Look into a module to find whether it contains a
386	Reader class. If so, append it to readers and (potentially)
387	to the list of writers for the given extension
388
389	:param ext: file extension [string]
390	:param module: module object
391	"""
392	return self.__registry.associate_file_type(ext, module)
393
394	def associate_file_reader(self, ext, loader):
395	"""
396	Append a reader object to readers
397
398	:param ext: file extension [string]
399	:param module: reader object
400	"""
401	return self.__registry.associate_file_reader(ext, loader)
402
403	def load(self, file, format=None):
404	"""
405	Load a file
406
407	:param file: file name (path)
408	:param format: specified format to use (optional)
409	:return: DataInfo object
410	"""
411	return self.__registry.load(file, format)
412
413	def save(self, file, data, format):
414	"""
415	Save a DataInfo object to file
416	:param file: file name (path)
417	:param data: DataInfo object
418	:param format: format to write the data in
419	"""
420	return self.__registry.save(file, data, format)
421
422	def _get_registry_creation_time(self):
423	"""
424	Internal method used to test the uniqueness
425	of the registry object
426	"""
427	return self.__registry._created
428
429	def find_plugins(self, directory):
430	"""
431	Find plugins in a given directory
432
433	:param dir: directory to look into to find new readers/writers
434	"""
435	return self.__registry.find_plugins(directory)
436
437	def get_wildcards(self):
438	"""
439	Return the list of wildcards
440	"""
441	return self.__registry.wildcards

Note: See TracBrowser for help on using the repository browser.

SasView

source: sasview/src/sas/sascalc/dataloader/loader.py @ 0e0c645

Download in other formats: