Changeset d321747 in sasmodels


Ignore:
Timestamp:
Oct 1, 2018 9:20:17 PM (3 weeks ago)
Author:
Paul Kienzle <pkienzle@…>
Branches:
master, beta_approx, cuda-test, py3, ticket-1074-gammainc, ticket-1157
Children:
6da1d76
Parents:
a5bcd61
Message:

sasview only updates models when code changes; now detects changes to c files

Location:
sasmodels
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • sasmodels/custom/__init__.py

    r91bd550 rd321747  
    1212import sys 
    1313import os 
    14 from os.path import basename, splitext 
     14from os.path import basename, splitext, join as joinpath, exists, dirname 
    1515 
    1616try: 
     
    1818    from importlib.util import spec_from_file_location, module_from_spec  # type: ignore 
    1919    def load_module_from_path(fullname, path): 
     20        # type: (str, str) -> "module" 
    2021        """load module from *path* as *fullname*""" 
    2122        spec = spec_from_file_location(fullname, os.path.expanduser(path)) 
     
    2728    import imp 
    2829    def load_module_from_path(fullname, path): 
     30        # type: (str, str) -> "module" 
    2931        """load module from *path* as *fullname*""" 
    3032        # Clear out old definitions, if any 
     
    3739        return module 
    3840 
    39 _MODULE_CACHE = {} 
    40 _MODULE_DEPENDS = {} 
    41 _MODULE_DEPENDS_STACK = [] 
     41_MODULE_CACHE = {} # type: Dict[str, Tuple("module", int)] 
     42_MODULE_DEPENDS = {} # type: Dict[str, List[str]] 
     43_MODULE_DEPENDS_STACK = [] # type: List[str] 
    4244def load_custom_kernel_module(path): 
     45    # type: str -> "module" 
    4346    """load SAS kernel from *path* as *sasmodels.custom.modelname*""" 
    4447    # Pull off the last .ext if it exists; there may be others 
     
    4649    path = os.path.expanduser(path) 
    4750 
    48     # reload module if necessary 
     51    # Reload module if necessary. 
    4952    if need_reload(path): 
    50         # Push to the next dependency level 
    51         _MODULE_DEPENDS_STACK.append(path) 
     53        # Assume the module file is the only dependency 
    5254        _MODULE_DEPENDS[path] = set([path]) 
    5355 
    54         # Load module into the 'sasmodels.custom' name space. 
    55         # If this triggers any submodule loads then they will be added 
    56         # as dependencies below when _MODULE_DEPENDS_STACK is not empty. 
     56        # Load the module while pushing it onto the dependency stack.  If 
     57        # this triggers any submodules, then they will add their dependencies 
     58        # to this module as the "working_on" parent.  Pop the stack when the 
     59        # module is loaded. 
     60        _MODULE_DEPENDS_STACK.append(path) 
    5761        module = load_module_from_path('sasmodels.custom.'+name, path) 
    58  
    59         # Pop the dependency level 
    6062        _MODULE_DEPENDS_STACK.pop() 
    6163 
    62         # TODO: include external C code in the dependencies 
    63         # If we had the model info structure we could do the following: 
    64         #    _MODEL_DEPENDS[path].extend(generate.model_sources(info)) 
    65         # but at this point all we have is the module.  Don't want to 
    66         # repeat the logic in modelinfo.make_model_info. 
     64        # Include external C code in the dependencies.  We are looking 
     65        # for module.source and assuming that it is a list of C source files 
     66        # relative to the module itself.  Any files that do not exist, 
     67        # such as those in the standard libraries, will be ignored. 
     68        # TODO: look in builtin module path for standard c sources 
     69        # TODO: share code with generate.model_sources 
     70        c_sources = getattr(module, 'source', None) 
     71        if isinstance(c_sources, (list, tuple)): 
     72            _MODULE_DEPENDS[path].update(_find_sources(path, c_sources)) 
    6773 
    68         # Cache the module with the newest timestamp 
     74        # Cache the module, and tag it with the newest timestamp 
    6975        timestamp = max(os.path.getmtime(f) for f in _MODULE_DEPENDS[path]) 
    7076        _MODULE_CACHE[path] = module, timestamp 
     
    7379        #    [os.path.basename(p) for p in _MODULE_DEPENDS[path]]) 
    7480 
     81    # Add path and all its dependence to the parent module, if there is one. 
    7582    if _MODULE_DEPENDS_STACK: 
    76         # Add child and all its dependence to the parent module 
    7783        working_on = _MODULE_DEPENDS_STACK[-1] 
    7884        _MODULE_DEPENDS[working_on].update(_MODULE_DEPENDS[path]) 
     
    8187 
    8288def need_reload(path): 
     89    # type: str -> bool 
     90    """ 
     91    Return True if any path dependencies have a timestamp newer than the time 
     92    when the path was most recently loaded. 
     93    """ 
    8394    # TODO: fails if a dependency has a modification time in the future 
    8495    # If the newest dependency has a time stamp in the future, then this 
     
    92103    _, cache_time = _MODULE_CACHE.get(path, (None, -1)) 
    93104    depends = _MODULE_DEPENDS.get(path, [path]) 
     105    #print("reload", any(cache_time < os.path.getmtime(p) for p in depends)) 
     106    #for f in depends: print(">>>  ", f, os.path.getmtime(f)) 
    94107    return any(cache_time < os.path.getmtime(p) for p in depends) 
     108 
     109def _find_sources(path, source_list): 
     110    # type: (str, List[str]) -> List[str] 
     111    """ 
     112    Return a list of the sources relative to base file; ignore any that 
     113    are not found. 
     114    """ 
     115    root = dirname(path) 
     116    found = [] 
     117    for source_name in source_list: 
     118        source_path = joinpath(root, source_name) 
     119        if exists(source_path): 
     120            found.append(source_path) 
     121    return found 
  • sasmodels/modelinfo.py

    r7b9e4dd rd321747  
    793793    info.structure_factor = getattr(kernel_module, 'structure_factor', False) 
    794794    info.profile_axes = getattr(kernel_module, 'profile_axes', ['x', 'y']) 
     795    # Note: custom.load_custom_kernel_module assumes the C sources are defined 
     796    # by this attribute. 
    795797    info.source = getattr(kernel_module, 'source', []) 
    796798    info.c_code = getattr(kernel_module, 'c_code', None) 
  • sasmodels/sasview_model.py

    r839fd68 rd321747  
    6565#: custom model {path: model} mapping so we can check timestamps 
    6666MODEL_BY_PATH = {}  # type: Dict[str, SasviewModelType] 
     67#: Track modules that we have loaded so we can determine whether the model 
     68#: has changed since we last reloaded. 
     69_CACHED_MODULE = {}  # type: Dict[str, "module"] 
    6770 
    6871def find_model(modelname): 
     
    108111    """ 
    109112    #logger.info("Loading model %s", path) 
     113 
     114    # Load the kernel module.  This may already be cached by the loader, so 
     115    # only requires checking the timestamps of the dependents. 
    110116    kernel_module = custom.load_custom_kernel_module(path) 
    111     if hasattr(kernel_module, 'Model'): 
    112         model = kernel_module.Model 
     117 
     118    # Check if the module has changed since we last looked. 
     119    reloaded = kernel_module != _CACHED_MODULE.get(path, None) 
     120    _CACHED_MODULE[path] = kernel_module 
     121 
     122    # Turn the module into a model.  We need to do this in even if the 
     123    # model has already been loaded so that we can determine the model 
     124    # name and retrieve it from the MODELS cache. 
     125    model = getattr(kernel_module, 'Model', None) 
     126    if model is not None: 
    113127        # Old style models do not set the name in the class attributes, so 
    114128        # set it here; this name will be overridden when the object is created 
     
    138152                    _previous_name, model.name, model.filename) 
    139153 
    140     MODELS[model.name] = model 
    141     return model 
     154    # Only update the model if the module has changed 
     155    if reloaded or model.name not in MODELS: 
     156        MODELS[model.name] = model 
     157 
     158    return MODELS[model.name] 
    142159 
    143160 
Note: See TracChangeset for help on using the changeset viewer.