source: sasview/src/sas/sascalc/dataloader/loader.py @ 278ddee

ESS_GUIESS_GUI_PrESS_GUI_cnsESS_GUI_corfESS_GUI_py2_OLDSVCC-1SasView-664costrafo411ticket-976unittest-saveload
Last change on this file since 278ddee was 278ddee, checked in by krzywon, 10 months ago

Merge branch 'master' into ticket-876

# Conflicts:
# src/sas/sascalc/dataloader/readers/ascii_reader.py
# src/sas/sascalc/dataloader/readers/xml_reader.py

  • Property mode set to 100644
File size: 14.1 KB
Line 
1"""
2    File handler to support different file extensions.
3    Uses reflectometer registry utility.
4
5    The default readers are found in the 'readers' sub-module
6    and registered by default at initialization time.
7
8    To add a new default reader, one must register it in
9    the register_readers method found in readers/__init__.py.
10
11    A utility method (find_plugins) is available to inspect
12    a directory (for instance, a user plug-in directory) and
13    look for new readers/writers.
14"""
15#####################################################################
16#This software was developed by the University of Tennessee as part of the
17#Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
18#project funded by the US National Science Foundation.
19#See the license text in license.txt
20#copyright 2008, University of Tennessee
21######################################################################
22
23import os
24import sys
25import logging
26import time
27from zipfile import ZipFile
28from sas.sascalc.data_util.registry import ExtensionRegistry
29# Default readers are defined in the readers sub-module
30import readers
31from loader_exceptions import NoKnownLoaderException, FileContentsException
32from readers import ascii_reader
33from readers import cansas_reader
34from readers import cansas_reader_HDF5
35
36
37logger = logging.getLogger(__name__)
38
39class Registry(ExtensionRegistry):
40    """
41    Registry class for file format extensions.
42    Readers and writers are supported.
43    """
44    def __init__(self):
45        super(Registry, self).__init__()
46
47        # Writers
48        self.writers = {}
49
50        # List of wildcards
51        self.wildcards = ['All (*.*)|*.*']
52
53        # Creation time, for testing
54        self._created = time.time()
55
56        # Register default readers
57        readers.read_associations(self)
58
59    def load(self, path, format=None):
60        """
61        Call the loader for the file type of path.
62
63        :param path: file path
64        :param format: explicit extension, to force the use
65            of a particular reader
66
67        Defaults to the ascii (multi-column), cansas XML, and cansas NeXuS
68        readers if no reader was registered for the file's extension.
69        """
70        try:
71            return super(Registry, self).load(path, format=format)
72        except NoKnownLoaderException as e:
73            pass  # try the ASCII reader
74        try:
75            ascii_loader = ascii_reader.Reader()
76            return ascii_loader.read(path)
77        except FileContentsException:
78            pass  # try the cansas XML reader
79        try:
80            cansas_loader = cansas_reader.Reader()
81            return cansas_loader.read(path)
82        except FileContentsException:
83            pass  # try the cansas NeXuS reader
84        try:
85            cansas_nexus_loader = cansas_reader_HDF5.Reader()
86            return cansas_nexus_loader.read(path)
87        except FileContentsException:
88            logging.errors("No default loader can load the data")
89            # No known reader available. Give up and throw an error
90            msg = "\n\tUnknown data format: %s.\n\tThe file is not a " % path
91            msg += "known format that can be loaded by SasView.\n"
92            msg += "Traceback:\n%s" % e.message
93            raise NoKnownLoaderException, msg
94
95    def find_plugins(self, dir):
96        """
97        Find readers in a given directory. This method
98        can be used to inspect user plug-in directories to
99        find new readers/writers.
100
101        :param dir: directory to search into
102        :return: number of readers found
103        """
104        readers_found = 0
105        temp_path = os.path.abspath(dir)
106        if not os.path.isdir(temp_path):
107            temp_path = os.path.join(os.getcwd(), dir)
108        if not os.path.isdir(temp_path):
109            temp_path = os.path.join(os.path.dirname(__file__), dir)
110        if not os.path.isdir(temp_path):
111            temp_path = os.path.join(os.path.dirname(sys.path[0]), dir)
112
113        dir = temp_path
114        # Check whether the directory exists
115        if not os.path.isdir(dir):
116            msg = "DataLoader couldn't locate DataLoader plugin folder."
117            msg += """ "%s" does not exist""" % dir
118            logger.warning(msg)
119            return readers_found
120
121        for item in os.listdir(dir):
122            full_path = os.path.join(dir, item)
123            if os.path.isfile(full_path):
124
125                # Process python files
126                if item.endswith('.py'):
127                    toks = os.path.splitext(os.path.basename(item))
128                    try:
129                        sys.path.insert(0, os.path.abspath(dir))
130                        module = __import__(toks[0], globals(), locals())
131                        if self._identify_plugin(module):
132                            readers_found += 1
133                    except:
134                        msg = "Loader: Error importing "
135                        msg += "%s\n  %s" % (item, sys.exc_value)
136                        logger.error(msg)
137
138                # Process zip files
139                elif item.endswith('.zip'):
140                    try:
141                        # Find the modules in the zip file
142                        zfile = ZipFile(item)
143                        nlist = zfile.namelist()
144
145                        sys.path.insert(0, item)
146                        for mfile in nlist:
147                            try:
148                                # Change OS path to python path
149                                fullname = mfile.replace('/', '.')
150                                fullname = os.path.splitext(fullname)[0]
151                                module = __import__(fullname, globals(),
152                                                    locals(), [""])
153                                if self._identify_plugin(module):
154                                    readers_found += 1
155                            except:
156                                msg = "Loader: Error importing"
157                                msg += " %s\n  %s" % (mfile, sys.exc_value)
158                                logger.error(msg)
159
160                    except:
161                        msg = "Loader: Error importing "
162                        msg += " %s\n  %s" % (item, sys.exc_value)
163                        logger.error(msg)
164
165        return readers_found
166
167    def associate_file_type(self, ext, module):
168        """
169        Look into a module to find whether it contains a
170        Reader class. If so, APPEND it to readers and (potentially)
171        to the list of writers for the given extension
172
173        :param ext: file extension [string]
174        :param module: module object
175        """
176        reader_found = False
177
178        if hasattr(module, "Reader"):
179            try:
180                # Find supported extensions
181                loader = module.Reader()
182                if ext not in self.loaders:
183                    self.loaders[ext] = []
184                # Append the new reader to the list
185                self.loaders[ext].append(loader.read)
186
187                reader_found = True
188
189                # Keep track of wildcards
190                type_name = module.__name__
191                if hasattr(loader, 'type_name'):
192                    type_name = loader.type_name
193
194                wcard = "%s files (*%s)|*%s" % (type_name, ext.lower(),
195                                                ext.lower())
196                if wcard not in self.wildcards:
197                    self.wildcards.append(wcard)
198
199                # Check whether writing is supported
200                if hasattr(loader, 'write'):
201                    if ext not in self.writers:
202                        self.writers[ext] = []
203                    # Append the new writer to the list
204                    self.writers[ext].append(loader.write)
205
206            except:
207                msg = "Loader: Error accessing"
208                msg += " Reader in %s\n  %s" % (module.__name__, sys.exc_value)
209                logger.error(msg)
210        return reader_found
211
212    def associate_file_reader(self, ext, loader):
213        """
214        Append a reader object to readers
215
216        :param ext: file extension [string]
217        :param module: reader object
218        """
219        reader_found = False
220
221        try:
222            # Find supported extensions
223            if ext not in self.loaders:
224                self.loaders[ext] = []
225            # Append the new reader to the list
226            self.loaders[ext].append(loader.read)
227
228            reader_found = True
229
230            # Keep track of wildcards
231            if hasattr(loader, 'type_name'):
232                type_name = loader.type_name
233
234                wcard = "%s files (*%s)|*%s" % (type_name, ext.lower(),
235                                                ext.lower())
236                if wcard not in self.wildcards:
237                    self.wildcards.append(wcard)
238
239        except:
240            msg = "Loader: Error accessing Reader "
241            msg += "in %s\n  %s" % (loader.__name__, sys.exc_value)
242            logger.error(msg)
243        return reader_found
244
245    def _identify_plugin(self, module):
246        """
247        Look into a module to find whether it contains a
248        Reader class. If so, add it to readers and (potentially)
249        to the list of writers.
250        :param module: module object
251
252        """
253        reader_found = False
254
255        if hasattr(module, "Reader"):
256            try:
257                # Find supported extensions
258                loader = module.Reader()
259                for ext in loader.ext:
260                    if ext not in self.loaders:
261                        self.loaders[ext] = []
262                    # When finding a reader at run time,
263                    # treat this reader as the new default
264                    self.loaders[ext].insert(0, loader.read)
265
266                    reader_found = True
267
268                    # Keep track of wildcards
269                    type_name = module.__name__
270                    if hasattr(loader, 'type_name'):
271                        type_name = loader.type_name
272                    wcard = "%s files (*%s)|*%s" % (type_name, ext.lower(),
273                                                    ext.lower())
274                    if wcard not in self.wildcards:
275                        self.wildcards.append(wcard)
276
277                # Check whether writing is supported
278                if hasattr(loader, 'write'):
279                    for ext in loader.ext:
280                        if ext not in self.writers:
281                            self.writers[ext] = []
282                        self.writers[ext].insert(0, loader.write)
283
284            except:
285                msg = "Loader: Error accessing Reader"
286                msg += " in %s\n  %s" % (module.__name__, sys.exc_value)
287                logger.error(msg)
288        return reader_found
289
290    def lookup_writers(self, path):
291        """
292        :return: the loader associated with the file type of path.
293        :Raises ValueError: if file type is not known.
294        """
295        # Find matching extensions
296        extlist = [ext for ext in self.extensions() if path.endswith(ext)]
297        # Sort matching extensions by decreasing order of length
298        extlist.sort(lambda a, b: len(a) < len(b))
299        # Combine loaders for matching extensions into one big list
300        writers = []
301        for L in [self.writers[ext] for ext in extlist]:
302            writers.extend(L)
303        # Remove duplicates if they exist
304        if len(writers) != len(set(writers)):
305            result = []
306            for L in writers:
307                if L not in result:
308                    result.append(L)
309            writers = L
310        # Raise an error if there are no matching extensions
311        if len(writers) == 0:
312            raise ValueError, "Unknown file type for " + path
313        # All done
314        return writers
315
316    def save(self, path, data, format=None):
317        """
318        Call the writer for the file type of path.
319
320        Raises ValueError if no writer is available.
321        Raises KeyError if format is not available.
322        May raise a writer-defined exception if writer fails.
323        """
324        if format is None:
325            writers = self.lookup_writers(path)
326        else:
327            writers = self.writers[format]
328        for fn in writers:
329            try:
330                return fn(path, data)
331            except:
332                pass  # give other loaders a chance to succeed
333        # If we get here it is because all loaders failed
334        raise  # reraises last exception
335
336
337class Loader(object):
338    """
339    Utility class to use the Registry as a singleton.
340    """
341    ## Registry instance
342    __registry = Registry()
343
344    def associate_file_type(self, ext, module):
345        """
346        Look into a module to find whether it contains a
347        Reader class. If so, append it to readers and (potentially)
348        to the list of writers for the given extension
349
350        :param ext: file extension [string]
351        :param module: module object
352        """
353        return self.__registry.associate_file_type(ext, module)
354
355    def associate_file_reader(self, ext, loader):
356        """
357        Append a reader object to readers
358
359        :param ext: file extension [string]
360        :param module: reader object
361        """
362        return self.__registry.associate_file_reader(ext, loader)
363
364    def load(self, file, format=None):
365        """
366        Load a file
367
368        :param file: file name (path)
369        :param format: specified format to use (optional)
370        :return: DataInfo object
371        """
372        return self.__registry.load(file, format)
373
374    def save(self, file, data, format):
375        """
376        Save a DataInfo object to file
377        :param file: file name (path)
378        :param data: DataInfo object
379        :param format: format to write the data in
380        """
381        return self.__registry.save(file, data, format)
382
383    def _get_registry_creation_time(self):
384        """
385        Internal method used to test the uniqueness
386        of the registry object
387        """
388        return self.__registry._created
389
390    def find_plugins(self, directory):
391        """
392        Find plugins in a given directory
393
394        :param dir: directory to look into to find new readers/writers
395        """
396        return self.__registry.find_plugins(directory)
397
398    def get_wildcards(self):
399        """
400        Return the list of wildcards
401        """
402        return self.__registry.wildcards
Note: See TracBrowser for help on using the repository browser.