source: sasview/src/sas/sascalc/dataloader/loader.py @ da8bb53

ESS_GUIESS_GUI_CategroyManagerESS_GUI_DocsESS_GUI_PrESS_GUI_corfESS_GUI_iss957ESS_GUI_iss959ESS_GUI_maskingESS_GUI_model_editorESS_GUI_py2_OLDESS_GUI_reportingSVCC-1SasView-664costrafo411ticket-1094-headlessticket-1111ticket-818ticket-976ticket885ticket885bunittest-saveloadwin64bit_conda_vm
Last change on this file since da8bb53 was da8bb53, checked in by krzywon, 15 months ago

Added a 4th data loader exception for generic readers that cannot open file.

  • Property mode set to 100644
File size: 14.5 KB
Line 
1"""
2    File handler to support different file extensions.
3    Uses reflectometer registry utility.
4
5    The default readers are found in the 'readers' sub-module
6    and registered by default at initialization time.
7
8    To add a new default reader, one must register it in
9    the register_readers method found in readers/__init__.py.
10
11    A utility method (find_plugins) is available to inspect
12    a directory (for instance, a user plug-in directory) and
13    look for new readers/writers.
14"""
15#####################################################################
16#This software was developed by the University of Tennessee as part of the
17#Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
18#project funded by the US National Science Foundation.
19#See the license text in license.txt
20#copyright 2008, University of Tennessee
21######################################################################
22
23import os
24import sys
25import logging
26import time
27import mimetypes
28from zipfile import ZipFile
29from sas.sascalc.data_util.registry import ExtensionRegistry
30# Default readers are defined in the readers sub-module
31import readers
32from loader_exceptions import NoKnownLoaderException, FileContentsException,\
33    DefaultReaderException
34from readers import ascii_reader
35from readers import cansas_reader
36from readers import cansas_reader_HDF5
37
38
39logger = logging.getLogger(__name__)
40
41class Registry(ExtensionRegistry):
42    """
43    Registry class for file format extensions.
44    Readers and writers are supported.
45    """
46    def __init__(self):
47        super(Registry, self).__init__()
48
49        # Writers
50        self.writers = {}
51
52        # List of wildcards
53        self.wildcards = ['All (*.*)|*.*']
54
55        # Creation time, for testing
56        self._created = time.time()
57
58        # Register default readers
59        readers.read_associations(self)
60
61    def load(self, path, format=None):
62        """
63        Call the loader for the file type of path.
64
65        :param path: file path
66        :param format: explicit extension, to force the use
67            of a particular reader
68
69        Defaults to the ascii (multi-column), cansas XML, and cansas NeXuS
70        readers if no reader was registered for the file's extension.
71        """
72        try:
73            return super(Registry, self).load(path, format=format)
74        except NoKnownLoaderException as e:
75            pass  # try the ASCII reader
76        try:
77            ascii_loader = ascii_reader.Reader()
78            return ascii_loader.read(path)
79        except DefaultReaderException:
80            pass  # Loader specific error to try the cansas XML reader
81        try:
82            cansas_loader = cansas_reader.Reader()
83            return cansas_loader.read(path)
84        except DefaultReaderException:
85            pass  # Loader specific error to try the cansas NeXuS reader
86        except FileContentsException:
87            # TODO: Handle errors properly
88            pass
89        except Exception as csr:
90            # TODO: Modify cansas reader to throw DefaultReaderException
91            pass
92        try:
93            cansas_nexus_loader = cansas_reader_HDF5.Reader()
94            return cansas_nexus_loader.read(path)
95        except DefaultReaderException:
96            logging.errors("No default loader can load the data")
97            # No known reader available. Give up and throw an error
98            msg = "\n\tUnknown data format: %s.\n\tThe file is not a " % path
99            msg += "known format that can be loaded by SasView.\n"
100            msg += "Traceback:\n%s" % e.message
101            raise NoKnownLoaderException, msg
102        except FileContentsException:
103            # TODO: Handle error(s) properly
104            pass
105
106    def find_plugins(self, dir):
107        """
108        Find readers in a given directory. This method
109        can be used to inspect user plug-in directories to
110        find new readers/writers.
111
112        :param dir: directory to search into
113        :return: number of readers found
114        """
115        readers_found = 0
116        temp_path = os.path.abspath(dir)
117        if not os.path.isdir(temp_path):
118            temp_path = os.path.join(os.getcwd(), dir)
119        if not os.path.isdir(temp_path):
120            temp_path = os.path.join(os.path.dirname(__file__), dir)
121        if not os.path.isdir(temp_path):
122            temp_path = os.path.join(os.path.dirname(sys.path[0]), dir)
123
124        dir = temp_path
125        # Check whether the directory exists
126        if not os.path.isdir(dir):
127            msg = "DataLoader couldn't locate DataLoader plugin folder."
128            msg += """ "%s" does not exist""" % dir
129            logger.warning(msg)
130            return readers_found
131
132        for item in os.listdir(dir):
133            full_path = os.path.join(dir, item)
134            if os.path.isfile(full_path):
135
136                # Process python files
137                if item.endswith('.py'):
138                    toks = os.path.splitext(os.path.basename(item))
139                    try:
140                        sys.path.insert(0, os.path.abspath(dir))
141                        module = __import__(toks[0], globals(), locals())
142                        if self._identify_plugin(module):
143                            readers_found += 1
144                    except:
145                        msg = "Loader: Error importing "
146                        msg += "%s\n  %s" % (item, sys.exc_value)
147                        logger.error(msg)
148
149                # Process zip files
150                elif item.endswith('.zip'):
151                    try:
152                        # Find the modules in the zip file
153                        zfile = ZipFile(item)
154                        nlist = zfile.namelist()
155
156                        sys.path.insert(0, item)
157                        for mfile in nlist:
158                            try:
159                                # Change OS path to python path
160                                fullname = mfile.replace('/', '.')
161                                fullname = os.path.splitext(fullname)[0]
162                                module = __import__(fullname, globals(),
163                                                    locals(), [""])
164                                if self._identify_plugin(module):
165                                    readers_found += 1
166                            except:
167                                msg = "Loader: Error importing"
168                                msg += " %s\n  %s" % (mfile, sys.exc_value)
169                                logger.error(msg)
170
171                    except:
172                        msg = "Loader: Error importing "
173                        msg += " %s\n  %s" % (item, sys.exc_value)
174                        logger.error(msg)
175
176        return readers_found
177
178    def associate_file_type(self, ext, module):
179        """
180        Look into a module to find whether it contains a
181        Reader class. If so, APPEND it to readers and (potentially)
182        to the list of writers for the given extension
183
184        :param ext: file extension [string]
185        :param module: module object
186        """
187        reader_found = False
188
189        if hasattr(module, "Reader"):
190            try:
191                # Find supported extensions
192                loader = module.Reader()
193                if ext not in self.loaders:
194                    self.loaders[ext] = []
195                # Append the new reader to the list
196                self.loaders[ext].append(loader.read)
197
198                reader_found = True
199
200                # Keep track of wildcards
201                type_name = module.__name__
202                if hasattr(loader, 'type_name'):
203                    type_name = loader.type_name
204
205                wcard = "%s files (*%s)|*%s" % (type_name, ext.lower(),
206                                                ext.lower())
207                if wcard not in self.wildcards:
208                    self.wildcards.append(wcard)
209
210                # Check whether writing is supported
211                if hasattr(loader, 'write'):
212                    if ext not in self.writers:
213                        self.writers[ext] = []
214                    # Append the new writer to the list
215                    self.writers[ext].append(loader.write)
216
217            except:
218                msg = "Loader: Error accessing"
219                msg += " Reader in %s\n  %s" % (module.__name__, sys.exc_value)
220                logger.error(msg)
221        return reader_found
222
223    def associate_file_reader(self, ext, loader):
224        """
225        Append a reader object to readers
226
227        :param ext: file extension [string]
228        :param module: reader object
229        """
230        reader_found = False
231
232        try:
233            # Find supported extensions
234            if ext not in self.loaders:
235                self.loaders[ext] = []
236            # Append the new reader to the list
237            self.loaders[ext].append(loader.read)
238
239            reader_found = True
240
241            # Keep track of wildcards
242            if hasattr(loader, 'type_name'):
243                type_name = loader.type_name
244
245                wcard = "%s files (*%s)|*%s" % (type_name, ext.lower(),
246                                                ext.lower())
247                if wcard not in self.wildcards:
248                    self.wildcards.append(wcard)
249
250        except:
251            msg = "Loader: Error accessing Reader "
252            msg += "in %s\n  %s" % (loader.__name__, sys.exc_value)
253            logger.error(msg)
254        return reader_found
255
256    def _identify_plugin(self, module):
257        """
258        Look into a module to find whether it contains a
259        Reader class. If so, add it to readers and (potentially)
260        to the list of writers.
261        :param module: module object
262
263        """
264        reader_found = False
265
266        if hasattr(module, "Reader"):
267            try:
268                # Find supported extensions
269                loader = module.Reader()
270                for ext in loader.ext:
271                    if ext not in self.loaders:
272                        self.loaders[ext] = []
273                    # When finding a reader at run time,
274                    # treat this reader as the new default
275                    self.loaders[ext].insert(0, loader.read)
276
277                    reader_found = True
278
279                    # Keep track of wildcards
280                    type_name = module.__name__
281                    if hasattr(loader, 'type_name'):
282                        type_name = loader.type_name
283                    wcard = "%s files (*%s)|*%s" % (type_name, ext.lower(),
284                                                    ext.lower())
285                    if wcard not in self.wildcards:
286                        self.wildcards.append(wcard)
287
288                # Check whether writing is supported
289                if hasattr(loader, 'write'):
290                    for ext in loader.ext:
291                        if ext not in self.writers:
292                            self.writers[ext] = []
293                        self.writers[ext].insert(0, loader.write)
294
295            except:
296                msg = "Loader: Error accessing Reader"
297                msg += " in %s\n  %s" % (module.__name__, sys.exc_value)
298                logger.error(msg)
299        return reader_found
300
301    def lookup_writers(self, path):
302        """
303        :return: the loader associated with the file type of path.
304        :Raises ValueError: if file type is not known.
305        """
306        # Find matching extensions
307        extlist = [ext for ext in self.extensions() if path.endswith(ext)]
308        # Sort matching extensions by decreasing order of length
309        extlist.sort(lambda a, b: len(a) < len(b))
310        # Combine loaders for matching extensions into one big list
311        writers = []
312        for L in [self.writers[ext] for ext in extlist]:
313            writers.extend(L)
314        # Remove duplicates if they exist
315        if len(writers) != len(set(writers)):
316            result = []
317            for L in writers:
318                if L not in result:
319                    result.append(L)
320            writers = L
321        # Raise an error if there are no matching extensions
322        if len(writers) == 0:
323            raise ValueError, "Unknown file type for " + path
324        # All done
325        return writers
326
327    def save(self, path, data, format=None):
328        """
329        Call the writer for the file type of path.
330
331        Raises ValueError if no writer is available.
332        Raises KeyError if format is not available.
333        May raise a writer-defined exception if writer fails.
334        """
335        if format is None:
336            writers = self.lookup_writers(path)
337        else:
338            writers = self.writers[format]
339        for fn in writers:
340            try:
341                return fn(path, data)
342            except:
343                pass  # give other loaders a chance to succeed
344        # If we get here it is because all loaders failed
345        raise  # reraises last exception
346
347
348class Loader(object):
349    """
350    Utility class to use the Registry as a singleton.
351    """
352    ## Registry instance
353    __registry = Registry()
354
355    def associate_file_type(self, ext, module):
356        """
357        Look into a module to find whether it contains a
358        Reader class. If so, append it to readers and (potentially)
359        to the list of writers for the given extension
360
361        :param ext: file extension [string]
362        :param module: module object
363        """
364        return self.__registry.associate_file_type(ext, module)
365
366    def associate_file_reader(self, ext, loader):
367        """
368        Append a reader object to readers
369
370        :param ext: file extension [string]
371        :param module: reader object
372        """
373        return self.__registry.associate_file_reader(ext, loader)
374
375    def load(self, file, format=None):
376        """
377        Load a file
378
379        :param file: file name (path)
380        :param format: specified format to use (optional)
381        :return: DataInfo object
382        """
383        return self.__registry.load(file, format)
384
385    def save(self, file, data, format):
386        """
387        Save a DataInfo object to file
388        :param file: file name (path)
389        :param data: DataInfo object
390        :param format: format to write the data in
391        """
392        return self.__registry.save(file, data, format)
393
394    def _get_registry_creation_time(self):
395        """
396        Internal method used to test the uniqueness
397        of the registry object
398        """
399        return self.__registry._created
400
401    def find_plugins(self, directory):
402        """
403        Find plugins in a given directory
404
405        :param dir: directory to look into to find new readers/writers
406        """
407        return self.__registry.find_plugins(directory)
408
409    def get_wildcards(self):
410        """
411        Return the list of wildcards
412        """
413        return self.__registry.wildcards
Note: See TracBrowser for help on using the repository browser.