source: sasview/src/sas/sascalc/dataloader/loader.py @ 574adc7

ESS_GUIESS_GUI_DocsESS_GUI_batch_fittingESS_GUI_bumps_abstractionESS_GUI_iss1116ESS_GUI_iss879ESS_GUI_iss959ESS_GUI_openclESS_GUI_orderingESS_GUI_sync_sascalcmagnetic_scattrelease-4.2.2ticket-1009ticket-1094-headlessticket-1242-2d-resolutionticket-1243ticket-1249ticket885unittest-saveload
Last change on this file since 574adc7 was 574adc7, checked in by Paul Kienzle <pkienzle@…>, 5 years ago

convert sascalc to python 2/3 syntax

  • Property mode set to 100644
File size: 15.4 KB
Line 
1"""
2    File handler to support different file extensions.
3    Uses reflectometer registry utility.
4
5    The default readers are found in the 'readers' sub-module
6    and registered by default at initialization time.
7
8    To add a new default reader, one must register it in
9    the register_readers method found in readers/__init__.py.
10
11    A utility method (find_plugins) is available to inspect
12    a directory (for instance, a user plug-in directory) and
13    look for new readers/writers.
14"""
15#####################################################################
16# This software was developed by the University of Tennessee as part of the
17# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
18# project funded by the US National Science Foundation.
19# See the license text in license.txt
20# copyright 2008, University of Tennessee
21######################################################################
22
23import os
24import sys
25import logging
26import time
27from zipfile import ZipFile
28
29from sas.sascalc.data_util.registry import ExtensionRegistry
30
31# Default readers are defined in the readers sub-module
32from . import readers
33from .loader_exceptions import NoKnownLoaderException, FileContentsException,\
34    DefaultReaderException
35from .readers import ascii_reader
36from .readers import cansas_reader
37from .readers import cansas_reader_HDF5
38
39logger = logging.getLogger(__name__)
40
41
42class Registry(ExtensionRegistry):
43    """
44    Registry class for file format extensions.
45    Readers and writers are supported.
46    """
47    def __init__(self):
48        super(Registry, self).__init__()
49
50        # Writers
51        self.writers = {}
52
53        # List of wildcards
54        self.wildcards = ['All (*.*)|*.*']
55
56        # Creation time, for testing
57        self._created = time.time()
58
59        # Register default readers
60        readers.read_associations(self)
61
62    def load(self, path, format=None):
63        """
64        Call the loader for the file type of path.
65
66        :param path: file path
67        :param format: explicit extension, to force the use
68            of a particular reader
69
70        Defaults to the ascii (multi-column), cansas XML, and cansas NeXuS
71        readers if no reader was registered for the file's extension.
72        """
73        # Gets set to a string if the file has an associated reader that fails
74        msg_from_reader = None
75        try:
76            return super(Registry, self).load(path, format=format)
77        except NoKnownLoaderException as nkl_e:
78            pass  # Try the ASCII reader
79        except FileContentsException as fc_exc:
80            # File has an associated reader but it failed.
81            # Save the error message to display later, but try the 3 default loaders
82            msg_from_reader = fc_exc.message
83        except Exception:
84            pass
85
86        # File has no associated reader, or the associated reader failed.
87        # Try the ASCII reader
88        try:
89            ascii_loader = ascii_reader.Reader()
90            return ascii_loader.read(path)
91        except DefaultReaderException:
92            pass  # Loader specific error to try the cansas XML reader
93        except FileContentsException as e:
94            if msg_from_reader is None:
95                raise RuntimeError(e.message)
96
97        # ASCII reader failed - try CanSAS xML reader
98        try:
99            cansas_loader = cansas_reader.Reader()
100            return cansas_loader.read(path)
101        except DefaultReaderException:
102            pass  # Loader specific error to try the NXcanSAS reader
103        except FileContentsException as e:
104            if msg_from_reader is None:
105                raise RuntimeError(e.message)
106        except Exception:
107            pass
108
109        # CanSAS XML reader failed - try NXcanSAS reader
110        try:
111            cansas_nexus_loader = cansas_reader_HDF5.Reader()
112            return cansas_nexus_loader.read(path)
113        except DefaultReaderException as e:
114            logging.error("No default loader can load the data")
115            # No known reader available. Give up and throw an error
116            if msg_from_reader is None:
117                msg = "\nUnknown data format: {}.\nThe file is not a ".format(path)
118                msg += "known format that can be loaded by SasView.\n"
119                raise NoKnownLoaderException(msg)
120            else:
121                # Associated reader and default readers all failed.
122                # Show error message from associated reader
123                raise RuntimeError(msg_from_reader)
124        except FileContentsException as e:
125            err_msg = msg_from_reader if msg_from_reader is not None else e.message
126            raise RuntimeError(err_msg)
127
128    def find_plugins(self, dir):
129        """
130        Find readers in a given directory. This method
131        can be used to inspect user plug-in directories to
132        find new readers/writers.
133
134        :param dir: directory to search into
135        :return: number of readers found
136        """
137        readers_found = 0
138        temp_path = os.path.abspath(dir)
139        if not os.path.isdir(temp_path):
140            temp_path = os.path.join(os.getcwd(), dir)
141        if not os.path.isdir(temp_path):
142            temp_path = os.path.join(os.path.dirname(__file__), dir)
143        if not os.path.isdir(temp_path):
144            temp_path = os.path.join(os.path.dirname(sys.path[0]), dir)
145
146        dir = temp_path
147        # Check whether the directory exists
148        if not os.path.isdir(dir):
149            msg = "DataLoader couldn't locate DataLoader plugin folder."
150            msg += """ "%s" does not exist""" % dir
151            logger.warning(msg)
152            return readers_found
153
154        for item in os.listdir(dir):
155            full_path = os.path.join(dir, item)
156            if os.path.isfile(full_path):
157
158                # Process python files
159                if item.endswith('.py'):
160                    toks = os.path.splitext(os.path.basename(item))
161                    try:
162                        sys.path.insert(0, os.path.abspath(dir))
163                        module = __import__(toks[0], globals(), locals())
164                        if self._identify_plugin(module):
165                            readers_found += 1
166                    except:
167                        msg = "Loader: Error importing "
168                        msg += "%s\n  %s" % (item, sys.exc_value)
169                        logger.error(msg)
170
171                # Process zip files
172                elif item.endswith('.zip'):
173                    try:
174                        # Find the modules in the zip file
175                        zfile = ZipFile(item)
176                        nlist = zfile.namelist()
177
178                        sys.path.insert(0, item)
179                        for mfile in nlist:
180                            try:
181                                # Change OS path to python path
182                                fullname = mfile.replace('/', '.')
183                                fullname = os.path.splitext(fullname)[0]
184                                module = __import__(fullname, globals(),
185                                                    locals(), [""])
186                                if self._identify_plugin(module):
187                                    readers_found += 1
188                            except:
189                                msg = "Loader: Error importing"
190                                msg += " %s\n  %s" % (mfile, sys.exc_value)
191                                logger.error(msg)
192
193                    except:
194                        msg = "Loader: Error importing "
195                        msg += " %s\n  %s" % (item, sys.exc_value)
196                        logger.error(msg)
197
198        return readers_found
199
200    def associate_file_type(self, ext, module):
201        """
202        Look into a module to find whether it contains a
203        Reader class. If so, APPEND it to readers and (potentially)
204        to the list of writers for the given extension
205
206        :param ext: file extension [string]
207        :param module: module object
208        """
209        reader_found = False
210
211        if hasattr(module, "Reader"):
212            try:
213                # Find supported extensions
214                loader = module.Reader()
215                if ext not in self.loaders:
216                    self.loaders[ext] = []
217                # Append the new reader to the list
218                self.loaders[ext].append(loader.read)
219
220                reader_found = True
221
222                # Keep track of wildcards
223                type_name = module.__name__
224                if hasattr(loader, 'type_name'):
225                    type_name = loader.type_name
226
227                wcard = "%s files (*%s)|*%s" % (type_name, ext.lower(),
228                                                ext.lower())
229                if wcard not in self.wildcards:
230                    self.wildcards.append(wcard)
231
232                # Check whether writing is supported
233                if hasattr(loader, 'write'):
234                    if ext not in self.writers:
235                        self.writers[ext] = []
236                    # Append the new writer to the list
237                    self.writers[ext].append(loader.write)
238
239            except:
240                msg = "Loader: Error accessing"
241                msg += " Reader in %s\n  %s" % (module.__name__, sys.exc_value)
242                logger.error(msg)
243        return reader_found
244
245    def associate_file_reader(self, ext, loader):
246        """
247        Append a reader object to readers
248
249        :param ext: file extension [string]
250        :param module: reader object
251        """
252        reader_found = False
253
254        try:
255            # Find supported extensions
256            if ext not in self.loaders:
257                self.loaders[ext] = []
258            # Append the new reader to the list
259            self.loaders[ext].append(loader.read)
260
261            reader_found = True
262
263            # Keep track of wildcards
264            if hasattr(loader, 'type_name'):
265                type_name = loader.type_name
266
267                wcard = "%s files (*%s)|*%s" % (type_name, ext.lower(),
268                                                ext.lower())
269                if wcard not in self.wildcards:
270                    self.wildcards.append(wcard)
271
272        except:
273            msg = "Loader: Error accessing Reader "
274            msg += "in %s\n  %s" % (loader.__name__, sys.exc_value)
275            logger.error(msg)
276        return reader_found
277
278    def _identify_plugin(self, module):
279        """
280        Look into a module to find whether it contains a
281        Reader class. If so, add it to readers and (potentially)
282        to the list of writers.
283        :param module: module object
284
285        """
286        reader_found = False
287
288        if hasattr(module, "Reader"):
289            try:
290                # Find supported extensions
291                loader = module.Reader()
292                for ext in loader.ext:
293                    if ext not in self.loaders:
294                        self.loaders[ext] = []
295                    # When finding a reader at run time,
296                    # treat this reader as the new default
297                    self.loaders[ext].insert(0, loader.read)
298
299                    reader_found = True
300
301                    # Keep track of wildcards
302                    type_name = module.__name__
303                    if hasattr(loader, 'type_name'):
304                        type_name = loader.type_name
305                    wcard = "%s files (*%s)|*%s" % (type_name, ext.lower(),
306                                                    ext.lower())
307                    if wcard not in self.wildcards:
308                        self.wildcards.append(wcard)
309
310                # Check whether writing is supported
311                if hasattr(loader, 'write'):
312                    for ext in loader.ext:
313                        if ext not in self.writers:
314                            self.writers[ext] = []
315                        self.writers[ext].insert(0, loader.write)
316
317            except:
318                msg = "Loader: Error accessing Reader"
319                msg += " in %s\n  %s" % (module.__name__, sys.exc_value)
320                logger.error(msg)
321        return reader_found
322
323    def lookup_writers(self, path):
324        """
325        :return: the loader associated with the file type of path.
326        :Raises ValueError: if file type is not known.
327        """
328        # Find matching extensions
329        extlist = [ext for ext in self.extensions() if path.endswith(ext)]
330        # Sort matching extensions by decreasing order of length
331        extlist.sort(lambda a, b: len(a) < len(b))
332        # Combine loaders for matching extensions into one big list
333        writers = []
334        for L in [self.writers[ext] for ext in extlist]:
335            writers.extend(L)
336        # Remove duplicates if they exist
337        if len(writers) != len(set(writers)):
338            result = []
339            for L in writers:
340                if L not in result:
341                    result.append(L)
342            writers = L
343        # Raise an error if there are no matching extensions
344        if len(writers) == 0:
345            raise ValueError("Unknown file type for " + path)
346        # All done
347        return writers
348
349    def save(self, path, data, format=None):
350        """
351        Call the writer for the file type of path.
352
353        Raises ValueError if no writer is available.
354        Raises KeyError if format is not available.
355        May raise a writer-defined exception if writer fails.
356        """
357        if format is None:
358            writers = self.lookup_writers(path)
359        else:
360            writers = self.writers[format]
361        for fn in writers:
362            try:
363                return fn(path, data)
364            except Exception:
365                pass  # give other loaders a chance to succeed
366        # If we get here it is because all loaders failed
367        raise  # reraises last exception
368
369
370class Loader(object):
371    """
372    Utility class to use the Registry as a singleton.
373    """
374    ## Registry instance
375    __registry = Registry()
376
377    def associate_file_type(self, ext, module):
378        """
379        Look into a module to find whether it contains a
380        Reader class. If so, append it to readers and (potentially)
381        to the list of writers for the given extension
382
383        :param ext: file extension [string]
384        :param module: module object
385        """
386        return self.__registry.associate_file_type(ext, module)
387
388    def associate_file_reader(self, ext, loader):
389        """
390        Append a reader object to readers
391
392        :param ext: file extension [string]
393        :param module: reader object
394        """
395        return self.__registry.associate_file_reader(ext, loader)
396
397    def load(self, file, format=None):
398        """
399        Load a file
400
401        :param file: file name (path)
402        :param format: specified format to use (optional)
403        :return: DataInfo object
404        """
405        return self.__registry.load(file, format)
406
407    def save(self, file, data, format):
408        """
409        Save a DataInfo object to file
410        :param file: file name (path)
411        :param data: DataInfo object
412        :param format: format to write the data in
413        """
414        return self.__registry.save(file, data, format)
415
416    def _get_registry_creation_time(self):
417        """
418        Internal method used to test the uniqueness
419        of the registry object
420        """
421        return self.__registry._created
422
423    def find_plugins(self, directory):
424        """
425        Find plugins in a given directory
426
427        :param dir: directory to look into to find new readers/writers
428        """
429        return self.__registry.find_plugins(directory)
430
431    def get_wildcards(self):
432        """
433        Return the list of wildcards
434        """
435        return self.__registry.wildcards
Note: See TracBrowser for help on using the repository browser.