Changeset d321747 in sasmodels

Timestamp:

Oct 1, 2018 9:20:17 PM (6 years ago)

Author:

Paul Kienzle <pkienzle@…>

Branches:

master, core_shell_microgels, magnetic_model, ticket-1257-vesicle-product, ticket_1156, ticket_1265_superball, ticket_822_more_unit_tests

Children:

c11d09f

Parents:

a5bcd61

Message:

sasview only updates models when code changes; now detects changes to c files

Location:

sasmodels

Files:

• custom/__init__.py (modified) (8 diffs)
• modelinfo.py (modified) (1 diff)
• sasview_model.py (modified) (3 diffs)

sasmodels/custom/__init__.py

@@ -12 +12 @@
 import sys
 import os
-from os.path import basename, splitext
+from os.path import basename, splitext, join as joinpath, exists, dirname
 
 try:
@@ -18 +18 @@
     from importlib.util import spec_from_file_location, module_from_spec  # type: ignore
     def load_module_from_path(fullname, path):
+        # type: (str, str) -> "module"
         """load module from *path* as *fullname*"""
         spec = spec_from_file_location(fullname, os.path.expanduser(path))
@@ -27 +28 @@
     import imp
     def load_module_from_path(fullname, path):
+        # type: (str, str) -> "module"
         """load module from *path* as *fullname*"""
         # Clear out old definitions, if any
@@ -37 +39 @@
         return module
 
-_MODULE_CACHE = {}
-_MODULE_DEPENDS = {}
-_MODULE_DEPENDS_STACK = []
+_MODULE_CACHE = {} # type: Dict[str, Tuple("module", int)]
+_MODULE_DEPENDS = {} # type: Dict[str, List[str]]
+_MODULE_DEPENDS_STACK = [] # type: List[str]
 def load_custom_kernel_module(path):
+    # type: str -> "module"
     """load SAS kernel from *path* as *sasmodels.custom.modelname*"""
     # Pull off the last .ext if it exists; there may be others
@@ -46 +49 @@
     path = os.path.expanduser(path)
 
-    # reload module if necessary
+    # Reload module if necessary.
     if need_reload(path):
-        # Push to the next dependency level
-        _MODULE_DEPENDS_STACK.append(path)
+        # Assume the module file is the only dependency
         _MODULE_DEPENDS[path] = set([path])
 
-        # Load module into the 'sasmodels.custom' name space.
-        # If this triggers any submodule loads then they will be added
-        # as dependencies below when _MODULE_DEPENDS_STACK is not empty.
+        # Load the module while pushing it onto the dependency stack.  If
+        # this triggers any submodules, then they will add their dependencies
+        # to this module as the "working_on" parent.  Pop the stack when the
+        # module is loaded.
+        _MODULE_DEPENDS_STACK.append(path)
         module = load_module_from_path('sasmodels.custom.'+name, path)
-
-        # Pop the dependency level
         _MODULE_DEPENDS_STACK.pop()
 
-        # TODO: include external C code in the dependencies
-        # If we had the model info structure we could do the following:
-        #    _MODEL_DEPENDS[path].extend(generate.model_sources(info))
-        # but at this point all we have is the module.  Don't want to
-        # repeat the logic in modelinfo.make_model_info.
+        # Include external C code in the dependencies.  We are looking
+        # for module.source and assuming that it is a list of C source files
+        # relative to the module itself.  Any files that do not exist,
+        # such as those in the standard libraries, will be ignored.
+        # TODO: look in builtin module path for standard c sources
+        # TODO: share code with generate.model_sources
+        c_sources = getattr(module, 'source', None)
+        if isinstance(c_sources, (list, tuple)):
+            _MODULE_DEPENDS[path].update(_find_sources(path, c_sources))
 
-        # Cache the module with the newest timestamp
+        # Cache the module, and tag it with the newest timestamp
         timestamp = max(os.path.getmtime(f) for f in _MODULE_DEPENDS[path])
         _MODULE_CACHE[path] = module, timestamp
@@ -73 +79 @@
         #    [os.path.basename(p) for p in _MODULE_DEPENDS[path]])
 
+    # Add path and all its dependence to the parent module, if there is one.
     if _MODULE_DEPENDS_STACK:
-        # Add child and all its dependence to the parent module
         working_on = _MODULE_DEPENDS_STACK[-1]
         _MODULE_DEPENDS[working_on].update(_MODULE_DEPENDS[path])
@@ -81 +87 @@
 
 def need_reload(path):
+    # type: str -> bool
+    """
+    Return True if any path dependencies have a timestamp newer than the time
+    when the path was most recently loaded.
+    """
     # TODO: fails if a dependency has a modification time in the future
     # If the newest dependency has a time stamp in the future, then this
@@ -92 +103 @@
     _, cache_time = _MODULE_CACHE.get(path, (None, -1))
     depends = _MODULE_DEPENDS.get(path, [path])
+    #print("reload", any(cache_time < os.path.getmtime(p) for p in depends))
+    #for f in depends: print(">>>  ", f, os.path.getmtime(f))
     return any(cache_time < os.path.getmtime(p) for p in depends)
+
+def _find_sources(path, source_list):
+    # type: (str, List[str]) -> List[str]
+    """
+    Return a list of the sources relative to base file; ignore any that
+    are not found.
+    """
+    root = dirname(path)
+    found = []
+    for source_name in source_list:
+        source_path = joinpath(root, source_name)
+        if exists(source_path):
+            found.append(source_path)
+    return found

sasmodels/modelinfo.py

@@ -793 +793 @@
     info.structure_factor = getattr(kernel_module, 'structure_factor', False)
     info.profile_axes = getattr(kernel_module, 'profile_axes', ['x', 'y'])
+    # Note: custom.load_custom_kernel_module assumes the C sources are defined
+    # by this attribute.
     info.source = getattr(kernel_module, 'source', [])
     info.c_code = getattr(kernel_module, 'c_code', None)

sasmodels/sasview_model.py

@@ -65 +65 @@
 #: custom model {path: model} mapping so we can check timestamps
 MODEL_BY_PATH = {}  # type: Dict[str, SasviewModelType]
+#: Track modules that we have loaded so we can determine whether the model
+#: has changed since we last reloaded.
+_CACHED_MODULE = {}  # type: Dict[str, "module"]
 
 def find_model(modelname):
@@ -108 +111 @@
     """
     #logger.info("Loading model %s", path)
+
+    # Load the kernel module.  This may already be cached by the loader, so
+    # only requires checking the timestamps of the dependents.
     kernel_module = custom.load_custom_kernel_module(path)
-    if hasattr(kernel_module, 'Model'):
-        model = kernel_module.Model
+
+    # Check if the module has changed since we last looked.
+    reloaded = kernel_module != _CACHED_MODULE.get(path, None)
+    _CACHED_MODULE[path] = kernel_module
+
+    # Turn the module into a model.  We need to do this in even if the
+    # model has already been loaded so that we can determine the model
+    # name and retrieve it from the MODELS cache.
+    model = getattr(kernel_module, 'Model', None)
+    if model is not None:
         # Old style models do not set the name in the class attributes, so
         # set it here; this name will be overridden when the object is created
@@ -138 +152 @@
                     _previous_name, model.name, model.filename)
 
-    MODELS[model.name] = model
-    return model
+    # Only update the model if the module has changed
+    if reloaded or model.name not in MODELS:
+        MODELS[model.name] = model
+
+    return MODELS[model.name]