loader.py @ 83db1cc

ESS_GUIESS_GUI_DocsESS_GUI_batch_fittingESS_GUI_bumps_abstractionESS_GUI_iss1116ESS_GUI_iss879ESS_GUI_iss959ESS_GUI_openclESS_GUI_orderingESS_GUI_sync_sascalcmagnetic_scattrelease-4.2.2ticket-1009ticket-1094-headlessticket-1242-2d-resolutionticket-1243ticket-1249ticket885unittest-saveload

Last change on this file since 83db1cc was dcb91cf, checked in by lewis, 7 years ago
Make suggested changes
Property mode set to `100644`
File size: 15.4 KB

Line
1	"""
2	File handler to support different file extensions.
3	Uses reflectometer registry utility.
4
5	The default readers are found in the 'readers' sub-module
6	and registered by default at initialization time.
7
8	To add a new default reader, one must register it in
9	the register_readers method found in readers/__init__.py.
10
11	A utility method (find_plugins) is available to inspect
12	a directory (for instance, a user plug-in directory) and
13	look for new readers/writers.
14	"""
15	#####################################################################
16	# This software was developed by the University of Tennessee as part of the
17	# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
18	# project funded by the US National Science Foundation.
19	# See the license text in license.txt
20	# copyright 2008, University of Tennessee
21	######################################################################
22
23	import os
24	import sys
25	import logging
26	import time
27	from zipfile import ZipFile
28	from sas.sascalc.data_util.registry import ExtensionRegistry
29	# Default readers are defined in the readers sub-module
30	import readers
31	from loader_exceptions import NoKnownLoaderException, FileContentsException,\
32	DefaultReaderException
33	from readers import ascii_reader
34	from readers import cansas_reader
35	from readers import cansas_reader_HDF5
36
37	logger = logging.getLogger(__name__)
38
39
40	class Registry(ExtensionRegistry):
41	"""
42	Registry class for file format extensions.
43	Readers and writers are supported.
44	"""
45	def __init__(self):
46	super(Registry, self).__init__()
47
48	# Writers
49	self.writers = {}
50
51	# List of wildcards
52	self.wildcards = ['All (.)\|.']
53
54	# Creation time, for testing
55	self._created = time.time()
56
57	# Register default readers
58	readers.read_associations(self)
59
60	def load(self, path, format=None):
61	"""
62	Call the loader for the file type of path.
63
64	:param path: file path
65	:param format: explicit extension, to force the use
66	of a particular reader
67
68	Defaults to the ascii (multi-column), cansas XML, and cansas NeXuS
69	readers if no reader was registered for the file's extension.
70	"""
71	# Gets set to a string if the file has an associated reader that fails
72	msg_from_reader = None
73	try:
74	return super(Registry, self).load(path, format=format)
75	except NoKnownLoaderException as nkl_e:
76	pass # Try the ASCII reader
77	except FileContentsException as fc_exc:
78	# File has an associated reader but it failed.
79	# Save the error message to display later, but try the 3 default loaders
80	msg_from_reader = fc_exc.message
81	except Exception:
82	pass
83
84	# File has no associated reader, or the associated reader failed.
85	# Try the ASCII reader
86	try:
87	ascii_loader = ascii_reader.Reader()
88	return ascii_loader.read(path)
89	except DefaultReaderException:
90	pass # Loader specific error to try the cansas XML reader
91	except FileContentsException as e:
92	if msg_from_reader is None:
93	raise RuntimeError(e.message)
94
95	# ASCII reader failed - try CanSAS xML reader
96	try:
97	cansas_loader = cansas_reader.Reader()
98	return cansas_loader.read(path)
99	except DefaultReaderException:
100	pass # Loader specific error to try the NXcanSAS reader
101	except FileContentsException as e:
102	if msg_from_reader is None:
103	raise RuntimeError(e.message)
104	except Exception:
105	pass
106
107	# CanSAS XML reader failed - try NXcanSAS reader
108	try:
109	cansas_nexus_loader = cansas_reader_HDF5.Reader()
110	return cansas_nexus_loader.read(path)
111	except DefaultReaderException as e:
112	logging.error("No default loader can load the data")
113	# No known reader available. Give up and throw an error
114	if msg_from_reader is None:
115	msg = "\nUnknown data format: {}.\nThe file is not a ".format(path)
116	msg += "known format that can be loaded by SasView.\n"
117	raise NoKnownLoaderException(msg)
118	else:
119	# Associated reader and default readers all failed.
120	# Show error message from associated reader
121	raise RuntimeError(msg_from_reader)
122	except FileContentsException as e:
123	err_msg = msg_from_reader if msg_from_reader is not None else e.message
124	raise RuntimeError(err_msg)
125
126	def find_plugins(self, dir):
127	"""
128	Find readers in a given directory. This method
129	can be used to inspect user plug-in directories to
130	find new readers/writers.
131
132	:param dir: directory to search into
133	:return: number of readers found
134	"""
135	readers_found = 0
136	temp_path = os.path.abspath(dir)
137	if not os.path.isdir(temp_path):
138	temp_path = os.path.join(os.getcwd(), dir)
139	if not os.path.isdir(temp_path):
140	temp_path = os.path.join(os.path.dirname(__file__), dir)
141	if not os.path.isdir(temp_path):
142	temp_path = os.path.join(os.path.dirname(sys.path[0]), dir)
143
144	dir = temp_path
145	# Check whether the directory exists
146	if not os.path.isdir(dir):
147	msg = "DataLoader couldn't locate DataLoader plugin folder."
148	msg += """ "%s" does not exist""" % dir
149	logger.warning(msg)
150	return readers_found
151
152	for item in os.listdir(dir):
153	full_path = os.path.join(dir, item)
154	if os.path.isfile(full_path):
155
156	# Process python files
157	if item.endswith('.py'):
158	toks = os.path.splitext(os.path.basename(item))
159	try:
160	sys.path.insert(0, os.path.abspath(dir))
161	module = __import__(toks[0], globals(), locals())
162	if self._identify_plugin(module):
163	readers_found += 1
164	except:
165	msg = "Loader: Error importing "
166	msg += "%s\n %s" % (item, sys.exc_value)
167	logger.error(msg)
168
169	# Process zip files
170	elif item.endswith('.zip'):
171	try:
172	# Find the modules in the zip file
173	zfile = ZipFile(item)
174	nlist = zfile.namelist()
175
176	sys.path.insert(0, item)
177	for mfile in nlist:
178	try:
179	# Change OS path to python path
180	fullname = mfile.replace('/', '.')
181	fullname = os.path.splitext(fullname)[0]
182	module = __import__(fullname, globals(),
183	locals(), [""])
184	if self._identify_plugin(module):
185	readers_found += 1
186	except:
187	msg = "Loader: Error importing"
188	msg += " %s\n %s" % (mfile, sys.exc_value)
189	logger.error(msg)
190
191	except:
192	msg = "Loader: Error importing "
193	msg += " %s\n %s" % (item, sys.exc_value)
194	logger.error(msg)
195
196	return readers_found
197
198	def associate_file_type(self, ext, module):
199	"""
200	Look into a module to find whether it contains a
201	Reader class. If so, APPEND it to readers and (potentially)
202	to the list of writers for the given extension
203
204	:param ext: file extension [string]
205	:param module: module object
206	"""
207	reader_found = False
208
209	if hasattr(module, "Reader"):
210	try:
211	# Find supported extensions
212	loader = module.Reader()
213	if ext not in self.loaders:
214	self.loaders[ext] = []
215	# Append the new reader to the list
216	self.loaders[ext].append(loader.read)
217
218	reader_found = True
219
220	# Keep track of wildcards
221	type_name = module.__name__
222	if hasattr(loader, 'type_name'):
223	type_name = loader.type_name
224
225	wcard = "%s files (%s)\|%s" % (type_name, ext.lower(),
226	ext.lower())
227	if wcard not in self.wildcards:
228	self.wildcards.append(wcard)
229
230	# Check whether writing is supported
231	if hasattr(loader, 'write'):
232	if ext not in self.writers:
233	self.writers[ext] = []
234	# Append the new writer to the list
235	self.writers[ext].append(loader.write)
236
237	except:
238	msg = "Loader: Error accessing"
239	msg += " Reader in %s\n %s" % (module.__name__, sys.exc_value)
240	logger.error(msg)
241	return reader_found
242
243	def associate_file_reader(self, ext, loader):
244	"""
245	Append a reader object to readers
246
247	:param ext: file extension [string]
248	:param module: reader object
249	"""
250	reader_found = False
251
252	try:
253	# Find supported extensions
254	if ext not in self.loaders:
255	self.loaders[ext] = []
256	# Append the new reader to the list
257	self.loaders[ext].append(loader.read)
258
259	reader_found = True
260
261	# Keep track of wildcards
262	if hasattr(loader, 'type_name'):
263	type_name = loader.type_name
264
265	wcard = "%s files (%s)\|%s" % (type_name, ext.lower(),
266	ext.lower())
267	if wcard not in self.wildcards:
268	self.wildcards.append(wcard)
269
270	except:
271	msg = "Loader: Error accessing Reader "
272	msg += "in %s\n %s" % (loader.__name__, sys.exc_value)
273	logger.error(msg)
274	return reader_found
275
276	def _identify_plugin(self, module):
277	"""
278	Look into a module to find whether it contains a
279	Reader class. If so, add it to readers and (potentially)
280	to the list of writers.
281	:param module: module object
282
283	"""
284	reader_found = False
285
286	if hasattr(module, "Reader"):
287	try:
288	# Find supported extensions
289	loader = module.Reader()
290	for ext in loader.ext:
291	if ext not in self.loaders:
292	self.loaders[ext] = []
293	# When finding a reader at run time,
294	# treat this reader as the new default
295	self.loaders[ext].insert(0, loader.read)
296
297	reader_found = True
298
299	# Keep track of wildcards
300	type_name = module.__name__
301	if hasattr(loader, 'type_name'):
302	type_name = loader.type_name
303	wcard = "%s files (%s)\|%s" % (type_name, ext.lower(),
304	ext.lower())
305	if wcard not in self.wildcards:
306	self.wildcards.append(wcard)
307
308	# Check whether writing is supported
309	if hasattr(loader, 'write'):
310	for ext in loader.ext:
311	if ext not in self.writers:
312	self.writers[ext] = []
313	self.writers[ext].insert(0, loader.write)
314
315	except:
316	msg = "Loader: Error accessing Reader"
317	msg += " in %s\n %s" % (module.__name__, sys.exc_value)
318	logger.error(msg)
319	return reader_found
320
321	def lookup_writers(self, path):
322	"""
323	:return: the loader associated with the file type of path.
324	:Raises ValueError: if file type is not known.
325	"""
326	# Find matching extensions
327	extlist = [ext for ext in self.extensions() if path.endswith(ext)]
328	# Sort matching extensions by decreasing order of length
329	extlist.sort(lambda a, b: len(a) < len(b))
330	# Combine loaders for matching extensions into one big list
331	writers = []
332	for L in [self.writers[ext] for ext in extlist]:
333	writers.extend(L)
334	# Remove duplicates if they exist
335	if len(writers) != len(set(writers)):
336	result = []
337	for L in writers:
338	if L not in result:
339	result.append(L)
340	writers = L
341	# Raise an error if there are no matching extensions
342	if len(writers) == 0:
343	raise ValueError, "Unknown file type for " + path
344	# All done
345	return writers
346
347	def save(self, path, data, format=None):
348	"""
349	Call the writer for the file type of path.
350
351	Raises ValueError if no writer is available.
352	Raises KeyError if format is not available.
353	May raise a writer-defined exception if writer fails.
354	"""
355	if format is None:
356	writers = self.lookup_writers(path)
357	else:
358	writers = self.writers[format]
359	for fn in writers:
360	try:
361	return fn(path, data)
362	except:
363	pass # give other loaders a chance to succeed
364	# If we get here it is because all loaders failed
365	raise # reraises last exception
366
367
368	class Loader(object):
369	"""
370	Utility class to use the Registry as a singleton.
371	"""
372	## Registry instance
373	__registry = Registry()
374
375	def associate_file_type(self, ext, module):
376	"""
377	Look into a module to find whether it contains a
378	Reader class. If so, append it to readers and (potentially)
379	to the list of writers for the given extension
380
381	:param ext: file extension [string]
382	:param module: module object
383	"""
384	return self.__registry.associate_file_type(ext, module)
385
386	def associate_file_reader(self, ext, loader):
387	"""
388	Append a reader object to readers
389
390	:param ext: file extension [string]
391	:param module: reader object
392	"""
393	return self.__registry.associate_file_reader(ext, loader)
394
395	def load(self, file, format=None):
396	"""
397	Load a file
398
399	:param file: file name (path)
400	:param format: specified format to use (optional)
401	:return: DataInfo object
402	"""
403	return self.__registry.load(file, format)
404
405	def save(self, file, data, format):
406	"""
407	Save a DataInfo object to file
408	:param file: file name (path)
409	:param data: DataInfo object
410	:param format: format to write the data in
411	"""
412	return self.__registry.save(file, data, format)
413
414	def _get_registry_creation_time(self):
415	"""
416	Internal method used to test the uniqueness
417	of the registry object
418	"""
419	return self.__registry._created
420
421	def find_plugins(self, directory):
422	"""
423	Find plugins in a given directory
424
425	:param dir: directory to look into to find new readers/writers
426	"""
427	return self.__registry.find_plugins(directory)
428
429	def get_wildcards(self):
430	"""
431	Return the list of wildcards
432	"""
433	return self.__registry.wildcards

Note: See TracBrowser for help on using the repository browser.

SasView

source: sasview/src/sas/sascalc/dataloader/loader.py @ 83db1cc

Download in other formats: