loader.py @ ef63dd9

Last change on this file since ef63dd9 was ef63dd9, checked in by krzywon, 7 years ago
Throw error without attempting to load in deprecated files.
Property mode set to `100644`
File size: 15.9 KB

Line
1	"""
2	File handler to support different file extensions.
3	Uses reflectometer registry utility.
4
5	The default readers are found in the 'readers' sub-module
6	and registered by default at initialization time.
7
8	To add a new default reader, one must register it in
9	the register_readers method found in readers/__init__.py.
10
11	A utility method (find_plugins) is available to inspect
12	a directory (for instance, a user plug-in directory) and
13	look for new readers/writers.
14	"""
15	#####################################################################
16	# This software was developed by the University of Tennessee as part of the
17	# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
18	# project funded by the US National Science Foundation.
19	# See the license text in license.txt
20	# copyright 2008, University of Tennessee
21	######################################################################
22
23	import os
24	import sys
25	import logging
26	import time
27	from zipfile import ZipFile
28
29	from sas.sascalc.data_util.registry import ExtensionRegistry
30
31	# Default readers are defined in the readers sub-module
32	from . import readers
33	from .loader_exceptions import NoKnownLoaderException, FileContentsException,\
34	DefaultReaderException
35	from .readers import ascii_reader
36	from .readers import cansas_reader
37	from .readers import cansas_reader_HDF5
38
39	logger = logging.getLogger(__name__)
40
41
42	class Registry(ExtensionRegistry):
43	"""
44	Registry class for file format extensions.
45	Readers and writers are supported.
46	"""
47	def __init__(self):
48	super(Registry, self).__init__()
49
50	# Writers
51	self.writers = {}
52
53	# List of wildcards
54	self.wildcards = ['All (.)\|.']
55
56	# Deprecated extensions
57	self.deprecated_extensions = ['.asc', '.ASC', '.nxs', '.NXS']
58
59	# Creation time, for testing
60	self._created = time.time()
61
62	# Register default readers
63	readers.read_associations(self)
64
65	def load(self, path, format=None):
66	"""
67	Call the loader for the file type of path.
68
69	:param path: file path
70	:param format: explicit extension, to force the use
71	of a particular reader
72
73	Defaults to the ascii (multi-column), cansas XML, and cansas NeXuS
74	readers if no reader was registered for the file's extension.
75	"""
76	# Gets set to a string if the file has an associated reader that fails
77	msg_from_reader = None
78	extlist = [ext for ext in self.deprecated_extensions if path.endswith(ext)]
79	if extlist:
80	msg = ("\rThe file you are attempting to load, {}, is no "
81	"longer loadable into SasView. Please use data in Q space."
82	.format(path))
83	raise RuntimeError(msg)
84	try:
85	return super(Registry, self).load(path, format=format)
86	#except Exception: raise # for debugging, don't use fallback loader
87	except NoKnownLoaderException as nkl_e:
88	pass # Try the ASCII reader
89	except FileContentsException as fc_exc:
90	# File has an associated reader but it failed.
91	# Save the error message to display later, but try the 3 default loaders
92	msg_from_reader = fc_exc.message
93	except Exception:
94	pass
95
96	# File has no associated reader, or the associated reader failed.
97	# Try the ASCII reader
98	try:
99	ascii_loader = ascii_reader.Reader()
100	return ascii_loader.read(path)
101	except DefaultReaderException:
102	pass # Loader specific error to try the cansas XML reader
103	except FileContentsException as e:
104	if msg_from_reader is None:
105	raise RuntimeError(e.message)
106
107	# ASCII reader failed - try CanSAS xML reader
108	try:
109	cansas_loader = cansas_reader.Reader()
110	return cansas_loader.read(path)
111	except DefaultReaderException:
112	pass # Loader specific error to try the NXcanSAS reader
113	except FileContentsException as e:
114	if msg_from_reader is None:
115	raise RuntimeError(e.message)
116	except Exception:
117	pass
118
119	# CanSAS XML reader failed - try NXcanSAS reader
120	try:
121	cansas_nexus_loader = cansas_reader_HDF5.Reader()
122	return cansas_nexus_loader.read(path)
123	except DefaultReaderException as e:
124	logging.error("No default loader can load the data")
125	# No known reader available. Give up and throw an error
126	if msg_from_reader is None:
127	msg = "\nUnknown data format: {}.\nThe file is not a ".format(path)
128	msg += "known format that can be loaded by SasView.\n"
129	raise NoKnownLoaderException(msg)
130	else:
131	# Associated reader and default readers all failed.
132	# Show error message from associated reader
133	raise RuntimeError(msg_from_reader)
134	except FileContentsException as e:
135	err_msg = msg_from_reader if msg_from_reader is not None else e.message
136	raise RuntimeError(err_msg)
137
138	def find_plugins(self, dir):
139	"""
140	Find readers in a given directory. This method
141	can be used to inspect user plug-in directories to
142	find new readers/writers.
143
144	:param dir: directory to search into
145	:return: number of readers found
146	"""
147	readers_found = 0
148	temp_path = os.path.abspath(dir)
149	if not os.path.isdir(temp_path):
150	temp_path = os.path.join(os.getcwd(), dir)
151	if not os.path.isdir(temp_path):
152	temp_path = os.path.join(os.path.dirname(__file__), dir)
153	if not os.path.isdir(temp_path):
154	temp_path = os.path.join(os.path.dirname(sys.path[0]), dir)
155
156	dir = temp_path
157	# Check whether the directory exists
158	if not os.path.isdir(dir):
159	msg = "DataLoader couldn't locate DataLoader plugin folder."
160	msg += """ "%s" does not exist""" % dir
161	logger.warning(msg)
162	return readers_found
163
164	for item in os.listdir(dir):
165	full_path = os.path.join(dir, item)
166	if os.path.isfile(full_path):
167
168	# Process python files
169	if item.endswith('.py'):
170	toks = os.path.splitext(os.path.basename(item))
171	try:
172	sys.path.insert(0, os.path.abspath(dir))
173	module = __import__(toks[0], globals(), locals())
174	if self._identify_plugin(module):
175	readers_found += 1
176	except:
177	msg = "Loader: Error importing "
178	msg += "%s\n %s" % (item, sys.exc_value)
179	logger.error(msg)
180
181	# Process zip files
182	elif item.endswith('.zip'):
183	try:
184	# Find the modules in the zip file
185	zfile = ZipFile(item)
186	nlist = zfile.namelist()
187
188	sys.path.insert(0, item)
189	for mfile in nlist:
190	try:
191	# Change OS path to python path
192	fullname = mfile.replace('/', '.')
193	fullname = os.path.splitext(fullname)[0]
194	module = __import__(fullname, globals(),
195	locals(), [""])
196	if self._identify_plugin(module):
197	readers_found += 1
198	except:
199	msg = "Loader: Error importing"
200	msg += " %s\n %s" % (mfile, sys.exc_value)
201	logger.error(msg)
202
203	except:
204	msg = "Loader: Error importing "
205	msg += " %s\n %s" % (item, sys.exc_value)
206	logger.error(msg)
207
208	return readers_found
209
210	def associate_file_type(self, ext, module):
211	"""
212	Look into a module to find whether it contains a
213	Reader class. If so, APPEND it to readers and (potentially)
214	to the list of writers for the given extension
215
216	:param ext: file extension [string]
217	:param module: module object
218	"""
219	reader_found = False
220
221	if hasattr(module, "Reader"):
222	try:
223	# Find supported extensions
224	loader = module.Reader()
225	if ext not in self.loaders:
226	self.loaders[ext] = []
227	# Append the new reader to the list
228	self.loaders[ext].append(loader.read)
229
230	reader_found = True
231
232	# Keep track of wildcards
233	type_name = module.__name__
234	if hasattr(loader, 'type_name'):
235	type_name = loader.type_name
236
237	wcard = "%s files (%s)\|%s" % (type_name, ext.lower(),
238	ext.lower())
239	if wcard not in self.wildcards:
240	self.wildcards.append(wcard)
241
242	# Check whether writing is supported
243	if hasattr(loader, 'write'):
244	if ext not in self.writers:
245	self.writers[ext] = []
246	# Append the new writer to the list
247	self.writers[ext].append(loader.write)
248
249	except:
250	msg = "Loader: Error accessing"
251	msg += " Reader in %s\n %s" % (module.__name__, sys.exc_value)
252	logger.error(msg)
253	return reader_found
254
255	def associate_file_reader(self, ext, loader):
256	"""
257	Append a reader object to readers
258
259	:param ext: file extension [string]
260	:param module: reader object
261	"""
262	reader_found = False
263
264	try:
265	# Find supported extensions
266	if ext not in self.loaders:
267	self.loaders[ext] = []
268	# Append the new reader to the list
269	self.loaders[ext].append(loader.read)
270
271	reader_found = True
272
273	# Keep track of wildcards
274	if hasattr(loader, 'type_name'):
275	type_name = loader.type_name
276
277	wcard = "%s files (%s)\|%s" % (type_name, ext.lower(),
278	ext.lower())
279	if wcard not in self.wildcards:
280	self.wildcards.append(wcard)
281
282	except:
283	msg = "Loader: Error accessing Reader "
284	msg += "in %s\n %s" % (loader.__name__, sys.exc_value)
285	logger.error(msg)
286	return reader_found
287
288	def _identify_plugin(self, module):
289	"""
290	Look into a module to find whether it contains a
291	Reader class. If so, add it to readers and (potentially)
292	to the list of writers.
293	:param module: module object
294
295	"""
296	reader_found = False
297
298	if hasattr(module, "Reader"):
299	try:
300	# Find supported extensions
301	loader = module.Reader()
302	for ext in loader.ext:
303	if ext not in self.loaders:
304	self.loaders[ext] = []
305	# When finding a reader at run time,
306	# treat this reader as the new default
307	self.loaders[ext].insert(0, loader.read)
308
309	reader_found = True
310
311	# Keep track of wildcards
312	type_name = module.__name__
313	if hasattr(loader, 'type_name'):
314	type_name = loader.type_name
315	wcard = "%s files (%s)\|%s" % (type_name, ext.lower(),
316	ext.lower())
317	if wcard not in self.wildcards:
318	self.wildcards.append(wcard)
319
320	# Check whether writing is supported
321	if hasattr(loader, 'write'):
322	for ext in loader.ext:
323	if ext not in self.writers:
324	self.writers[ext] = []
325	self.writers[ext].insert(0, loader.write)
326
327	except:
328	msg = "Loader: Error accessing Reader"
329	msg += " in %s\n %s" % (module.__name__, sys.exc_value)
330	logger.error(msg)
331	return reader_found
332
333	def lookup_writers(self, path):
334	"""
335	:return: the loader associated with the file type of path.
336	:Raises ValueError: if file type is not known.
337	"""
338	# Find matching extensions
339	extlist = [ext for ext in self.extensions() if path.endswith(ext)]
340	# Sort matching extensions by decreasing order of length
341	extlist.sort(key=len)
342	# Combine loaders for matching extensions into one big list
343	writers = []
344	for L in [self.writers[ext] for ext in extlist]:
345	writers.extend(L)
346	# Remove duplicates if they exist
347	if len(writers) != len(set(writers)):
348	result = []
349	for L in writers:
350	if L not in result:
351	result.append(L)
352	writers = L
353	# Raise an error if there are no matching extensions
354	if len(writers) == 0:
355	raise ValueError("Unknown file type for " + path)
356	# All done
357	return writers
358
359	def save(self, path, data, format=None):
360	"""
361	Call the writer for the file type of path.
362
363	Raises ValueError if no writer is available.
364	Raises KeyError if format is not available.
365	May raise a writer-defined exception if writer fails.
366	"""
367	if format is None:
368	writers = self.lookup_writers(path)
369	else:
370	writers = self.writers[format]
371	for fn in writers:
372	try:
373	return fn(path, data)
374	except Exception:
375	pass # give other loaders a chance to succeed
376	# If we get here it is because all loaders failed
377	raise # reraises last exception
378
379
380	class Loader(object):
381	"""
382	Utility class to use the Registry as a singleton.
383	"""
384	## Registry instance
385	__registry = Registry()
386
387	def associate_file_type(self, ext, module):
388	"""
389	Look into a module to find whether it contains a
390	Reader class. If so, append it to readers and (potentially)
391	to the list of writers for the given extension
392
393	:param ext: file extension [string]
394	:param module: module object
395	"""
396	return self.__registry.associate_file_type(ext, module)
397
398	def associate_file_reader(self, ext, loader):
399	"""
400	Append a reader object to readers
401
402	:param ext: file extension [string]
403	:param module: reader object
404	"""
405	return self.__registry.associate_file_reader(ext, loader)
406
407	def load(self, file, format=None):
408	"""
409	Load a file
410
411	:param file: file name (path)
412	:param format: specified format to use (optional)
413	:return: DataInfo object
414	"""
415	return self.__registry.load(file, format)
416
417	def save(self, file, data, format):
418	"""
419	Save a DataInfo object to file
420	:param file: file name (path)
421	:param data: DataInfo object
422	:param format: format to write the data in
423	"""
424	return self.__registry.save(file, data, format)
425
426	def _get_registry_creation_time(self):
427	"""
428	Internal method used to test the uniqueness
429	of the registry object
430	"""
431	return self.__registry._created
432
433	def find_plugins(self, directory):
434	"""
435	Find plugins in a given directory
436
437	:param dir: directory to look into to find new readers/writers
438	"""
439	return self.__registry.find_plugins(directory)
440
441	def get_wildcards(self):
442	"""
443	Return the list of wildcards
444	"""
445	return self.__registry.wildcards

Note: See TracBrowser for help on using the repository browser.

SasView

source: sasview/src/sas/sascalc/dataloader/loader.py @ ef63dd9

Download in other formats: