source: sasview/src/sas/sascalc/dataloader/loader.py @ 270c882b

ESS_GUIESS_GUI_DocsESS_GUI_batch_fittingESS_GUI_bumps_abstractionESS_GUI_iss1116ESS_GUI_iss879ESS_GUI_iss959ESS_GUI_openclESS_GUI_orderingESS_GUI_sync_sascalccostrafo411magnetic_scattrelease-4.2.2ticket-1009ticket-1094-headlessticket-1242-2d-resolutionticket-1243ticket-1249ticket885unittest-saveload
Last change on this file since 270c882b was 270c882b, checked in by krzywon, 7 years ago

Beginning implementation of exception handling within data loader.

  • Property mode set to 100644
File size: 14.0 KB
Line 
1"""
2    File handler to support different file extensions.
3    Uses reflectometer registry utility.
4
5    The default readers are found in the 'readers' sub-module
6    and registered by default at initialization time.
7
8    To add a new default reader, one must register it in
9    the register_readers method found in readers/__init__.py.
10
11    A utility method (find_plugins) is available to inspect
12    a directory (for instance, a user plug-in directory) and
13    look for new readers/writers.
14"""
15#####################################################################
16#This software was developed by the University of Tennessee as part of the
17#Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
18#project funded by the US National Science Foundation.
19#See the license text in license.txt
20#copyright 2008, University of Tennessee
21######################################################################
22
23import os
24import sys
25import logging
26import time
27from zipfile import ZipFile
28from sas.sascalc.data_util.registry import ExtensionRegistry
29# Default readers are defined in the readers sub-module
30import readers
31from loader_exceptions import NoKnownLoaderException, FileContentsException
32from readers import ascii_reader
33from readers import cansas_reader
34from readers import cansas_reader_HDF5
35
36
37class Registry(ExtensionRegistry):
38    """
39    Registry class for file format extensions.
40    Readers and writers are supported.
41    """
42    def __init__(self):
43        super(Registry, self).__init__()
44
45        # Writers
46        self.writers = {}
47
48        # List of wildcards
49        self.wildcards = ['All (*.*)|*.*']
50
51        # Creation time, for testing
52        self._created = time.time()
53
54        # Register default readers
55        readers.read_associations(self)
56
57    def load(self, path, format=None):
58        """
59        Call the loader for the file type of path.
60
61        :param path: file path
62        :param format: explicit extension, to force the use
63            of a particular reader
64
65        Defaults to the ascii (multi-column), cansas XML, and cansas NeXuS
66        readers if no reader was registered for the file's extension.
67        """
68        try:
69            return super(Registry, self).load(path, format=format)
70        except NoKnownLoaderException as e:
71            pass  # try the ASCII reader
72        except FileContentsException as e:
73            pass
74        try:
75            ascii_loader = ascii_reader.Reader()
76            return ascii_loader.read(path)
77        except FileContentsException:
78            pass  # try the cansas XML reader
79        try:
80            cansas_loader = cansas_reader.Reader()
81            return cansas_loader.read(path)
82        except FileContentsException:
83            pass  # try the cansas NeXuS reader
84        try:
85            cansas_nexus_loader = cansas_reader_HDF5.Reader()
86            return cansas_nexus_loader.read(path)
87        except FileContentsException:
88            # No known reader available. Give up and throw an error
89            msg = "\n\tUnknown data format: %s.\n\tThe file is not a " % path
90            msg += "known format that can be loaded by SasView."
91            raise NoKnownLoaderException(msg)
92
93    def find_plugins(self, dir):
94        """
95        Find readers in a given directory. This method
96        can be used to inspect user plug-in directories to
97        find new readers/writers.
98
99        :param dir: directory to search into
100        :return: number of readers found
101        """
102        readers_found = 0
103        temp_path = os.path.abspath(dir)
104        if not os.path.isdir(temp_path):
105            temp_path = os.path.join(os.getcwd(), dir)
106        if not os.path.isdir(temp_path):
107            temp_path = os.path.join(os.path.dirname(__file__), dir)
108        if not os.path.isdir(temp_path):
109            temp_path = os.path.join(os.path.dirname(sys.path[0]), dir)
110
111        dir = temp_path
112        # Check whether the directory exists
113        if not os.path.isdir(dir):
114            msg = "DataLoader couldn't locate DataLoader plugin folder."
115            msg += """ "%s" does not exist""" % dir
116            logging.warning(msg)
117            return readers_found
118
119        for item in os.listdir(dir):
120            full_path = os.path.join(dir, item)
121            if os.path.isfile(full_path):
122
123                # Process python files
124                if item.endswith('.py'):
125                    toks = os.path.splitext(os.path.basename(item))
126                    try:
127                        sys.path.insert(0, os.path.abspath(dir))
128                        module = __import__(toks[0], globals(), locals())
129                        if self._identify_plugin(module):
130                            readers_found += 1
131                    except:
132                        msg = "Loader: Error importing "
133                        msg += "%s\n  %s" % (item, sys.exc_value)
134                        logging.error(msg)
135
136                # Process zip files
137                elif item.endswith('.zip'):
138                    try:
139                        # Find the modules in the zip file
140                        zfile = ZipFile(item)
141                        nlist = zfile.namelist()
142
143                        sys.path.insert(0, item)
144                        for mfile in nlist:
145                            try:
146                                # Change OS path to python path
147                                fullname = mfile.replace('/', '.')
148                                fullname = os.path.splitext(fullname)[0]
149                                module = __import__(fullname, globals(),
150                                                    locals(), [""])
151                                if self._identify_plugin(module):
152                                    readers_found += 1
153                            except:
154                                msg = "Loader: Error importing"
155                                msg += " %s\n  %s" % (mfile, sys.exc_value)
156                                logging.error(msg)
157
158                    except:
159                        msg = "Loader: Error importing "
160                        msg += " %s\n  %s" % (item, sys.exc_value)
161                        logging.error(msg)
162
163        return readers_found
164
165    def associate_file_type(self, ext, module):
166        """
167        Look into a module to find whether it contains a
168        Reader class. If so, APPEND it to readers and (potentially)
169        to the list of writers for the given extension
170
171        :param ext: file extension [string]
172        :param module: module object
173        """
174        reader_found = False
175
176        if hasattr(module, "Reader"):
177            try:
178                # Find supported extensions
179                loader = module.Reader()
180                if ext not in self.loaders:
181                    self.loaders[ext] = []
182                # Append the new reader to the list
183                self.loaders[ext].append(loader.read)
184
185                reader_found = True
186
187                # Keep track of wildcards
188                type_name = module.__name__
189                if hasattr(loader, 'type_name'):
190                    type_name = loader.type_name
191
192                wcard = "%s files (*%s)|*%s" % (type_name, ext.lower(),
193                                                ext.lower())
194                if wcard not in self.wildcards:
195                    self.wildcards.append(wcard)
196
197                # Check whether writing is supported
198                if hasattr(loader, 'write'):
199                    if ext not in self.writers:
200                        self.writers[ext] = []
201                    # Append the new writer to the list
202                    self.writers[ext].append(loader.write)
203
204            except:
205                msg = "Loader: Error accessing"
206                msg += " Reader in %s\n  %s" % (module.__name__, sys.exc_value)
207                logging.error(msg)
208        return reader_found
209
210    def associate_file_reader(self, ext, loader):
211        """
212        Append a reader object to readers
213
214        :param ext: file extension [string]
215        :param module: reader object
216        """
217        reader_found = False
218
219        try:
220            # Find supported extensions
221            if ext not in self.loaders:
222                self.loaders[ext] = []
223            # Append the new reader to the list
224            self.loaders[ext].append(loader.read)
225
226            reader_found = True
227
228            # Keep track of wildcards
229            if hasattr(loader, 'type_name'):
230                type_name = loader.type_name
231
232                wcard = "%s files (*%s)|*%s" % (type_name, ext.lower(),
233                                                ext.lower())
234                if wcard not in self.wildcards:
235                    self.wildcards.append(wcard)
236
237        except:
238            msg = "Loader: Error accessing Reader "
239            msg += "in %s\n  %s" % (loader.__name__, sys.exc_value)
240            logging.error(msg)
241        return reader_found
242
243    def _identify_plugin(self, module):
244        """
245        Look into a module to find whether it contains a
246        Reader class. If so, add it to readers and (potentially)
247        to the list of writers.
248        :param module: module object
249
250        """
251        reader_found = False
252
253        if hasattr(module, "Reader"):
254            try:
255                # Find supported extensions
256                loader = module.Reader()
257                for ext in loader.ext:
258                    if ext not in self.loaders:
259                        self.loaders[ext] = []
260                    # When finding a reader at run time,
261                    # treat this reader as the new default
262                    self.loaders[ext].insert(0, loader.read)
263
264                    reader_found = True
265
266                    # Keep track of wildcards
267                    type_name = module.__name__
268                    if hasattr(loader, 'type_name'):
269                        type_name = loader.type_name
270                    wcard = "%s files (*%s)|*%s" % (type_name, ext.lower(),
271                                                    ext.lower())
272                    if wcard not in self.wildcards:
273                        self.wildcards.append(wcard)
274
275                # Check whether writing is supported
276                if hasattr(loader, 'write'):
277                    for ext in loader.ext:
278                        if ext not in self.writers:
279                            self.writers[ext] = []
280                        self.writers[ext].insert(0, loader.write)
281
282            except:
283                msg = "Loader: Error accessing Reader"
284                msg += " in %s\n  %s" % (module.__name__, sys.exc_value)
285                logging.error(msg)
286        return reader_found
287
288    def lookup_writers(self, path):
289        """
290        :return: the loader associated with the file type of path.
291        :Raises ValueError: if file type is not known.
292        """
293        # Find matching extensions
294        extlist = [ext for ext in self.extensions() if path.endswith(ext)]
295        # Sort matching extensions by decreasing order of length
296        extlist.sort(lambda a, b: len(a) < len(b))
297        # Combine loaders for matching extensions into one big list
298        writers = []
299        for L in [self.writers[ext] for ext in extlist]:
300            writers.extend(L)
301        # Remove duplicates if they exist
302        if len(writers) != len(set(writers)):
303            result = []
304            for L in writers:
305                if L not in result:
306                    result.append(L)
307            writers = L
308        # Raise an error if there are no matching extensions
309        if len(writers) == 0:
310            raise ValueError, "Unknown file type for " + path
311        # All done
312        return writers
313
314    def save(self, path, data, format=None):
315        """
316        Call the writer for the file type of path.
317
318        Raises ValueError if no writer is available.
319        Raises KeyError if format is not available.
320        May raise a writer-defined exception if writer fails.
321        """
322        if format is None:
323            writers = self.lookup_writers(path)
324        else:
325            writers = self.writers[format]
326        for fn in writers:
327            try:
328                return fn(path, data)
329            except:
330                pass  # give other loaders a chance to succeed
331        # If we get here it is because all loaders failed
332        raise  # reraises last exception
333
334
335class Loader(object):
336    """
337    Utility class to use the Registry as a singleton.
338    """
339    ## Registry instance
340    __registry = Registry()
341
342    def associate_file_type(self, ext, module):
343        """
344        Look into a module to find whether it contains a
345        Reader class. If so, append it to readers and (potentially)
346        to the list of writers for the given extension
347
348        :param ext: file extension [string]
349        :param module: module object
350        """
351        return self.__registry.associate_file_type(ext, module)
352
353    def associate_file_reader(self, ext, loader):
354        """
355        Append a reader object to readers
356
357        :param ext: file extension [string]
358        :param module: reader object
359        """
360        return self.__registry.associate_file_reader(ext, loader)
361
362    def load(self, file, format=None):
363        """
364        Load a file
365
366        :param file: file name (path)
367        :param format: specified format to use (optional)
368        :return: DataInfo object
369        """
370        return self.__registry.load(file, format)
371
372    def save(self, file, data, format):
373        """
374        Save a DataInfo object to file
375        :param file: file name (path)
376        :param data: DataInfo object
377        :param format: format to write the data in
378        """
379        return self.__registry.save(file, data, format)
380
381    def _get_registry_creation_time(self):
382        """
383        Internal method used to test the uniqueness
384        of the registry object
385        """
386        return self.__registry._created
387
388    def find_plugins(self, directory):
389        """
390        Find plugins in a given directory
391
392        :param dir: directory to look into to find new readers/writers
393        """
394        return self.__registry.find_plugins(directory)
395
396    def get_wildcards(self):
397        """
398        Return the list of wildcards
399        """
400        return self.__registry.wildcards
Note: See TracBrowser for help on using the repository browser.