Changeset 952ea1f in sasview

Timestamp:

Oct 11, 2018 1:33:22 PM (6 years ago)

Author:

Paul Kienzle <pkienzle@…>

Branches:

master, magnetic_scatt, release-4.2.2, ticket-1009, ticket-1094-headless, ticket-1242-2d-resolution, ticket-1243, ticket-1249

Children:

7ba6470

Parents:

67ed543

git-author:

Paul Kienzle <pkienzle@…> (10/11/18 13:22:07)

git-committer:

Paul Kienzle <pkienzle@…> (10/11/18 13:33:22)

Message:

move c extensions from core/module.so to _module.so

Files:

• run.py (modified) (4 diffs)
• setup.py (modified) (6 diffs)
• src/sas/sascalc/calculator/c_extensions/__init__.py (deleted)
• src/sas/sascalc/calculator/c_extensions/sld2i_module.c (modified) (2 diffs)
• src/sas/sascalc/calculator/sas_gen.py (modified) (5 diffs)
• src/sas/sascalc/file_converter/bsl_loader.py (modified) (2 diffs)
• src/sas/sascalc/file_converter/c_ext/__init__.py (deleted)
• src/sas/sascalc/file_converter/c_ext/bsl_loader.c (modified) (2 diffs)
• src/sas/sascalc/pr/c_extensions/Cinvertor.c (modified) (3 diffs)
• src/sas/sascalc/pr/c_extensions/__init__.py (deleted)
• src/sas/sascalc/pr/invertor.py (modified) (10 diffs)

run.py

@@ -67 +67 @@
 
 
-def prepare():
+def prepare(rebuild=True):
     # Don't create *.pyc files
     sys.dont_write_bytecode = True
@@ -95 +95 @@
     try:
         import periodictable
-    except:
+    except ImportError:
         addpath(joinpath(root, '..', 'periodictable'))
 
     try:
         import bumps
-    except:
+    except ImportError:
         addpath(joinpath(root, '..', 'bumps'))
 
     try:
         import tinycc
-    except:
+    except ImportError:
         addpath(joinpath(root, '../tinycc/build/lib'))
 
@@ -111 +111 @@
     #addpath(os.path.join(root, '..','wxPython-src-3.0.0.0','wxPython'))
 
-    # Build project if the build directory does not already exist.
-    # PAK: with "update" we can always build since it is fast
-    if True or not os.path.exists(build_path):
+    # Put the sas source tree on the path
+    addpath(joinpath(root, 'src'))
+
+    # Put sasmodels on the path
+    addpath(joinpath(root, '../sasmodels/'))
+
+    # Check if the C extensions are already built
+    try:
+        from sas.sascalc.pr import _pr_inversion
+        from sas.sascalc.calculator import _sld2i
+        from sas.sascalc.file_converter import _bsl_loader
+    except ImportError:
+        rebuild = True
+
+    # Build C extensions if necessary.  Do an inplace build to simplify path.
+    if rebuild:
         import subprocess
-        build_cmd = [sys.executable, "setup.py", "build", "update"]
+        build_cmd = [sys.executable, "setup.py", "build_ext", "--inplace", "update"]
         if os.name == 'nt':
             build_cmd.append('--compiler=tinycc')
@@ -122 +135 @@
         with cd(root):
             subprocess.call(build_cmd, shell=shell)
-
-    # Put the source trees on the path
-    addpath(joinpath(root, 'src'))
-
-    # sasmodels on the path
-    addpath(joinpath(root, '../sasmodels/'))
-
-    # The sas.models package Compiled Model files should be pulled in from the build directory even though
-    # the source is stored in src/sas/models.
-
-    # Compiled modules need to be pulled from the build directory.
-    # Some packages are not where they are needed, so load them explicitly.
-    import sas.sascalc.pr
-    sas.sascalc.pr.core = import_package('sas.sascalc.pr.core',
-                                         joinpath(build_path, 'sas', 'sascalc', 'pr', 'core'))
-
-    # Compiled modules need to be pulled from the build directory.
-    # Some packages are not where they are needed, so load them explicitly.
-    import sas.sascalc.file_converter
-    sas.sascalc.file_converter.core = import_package('sas.sascalc.file_converter.core',
-                                                     joinpath(build_path, 'sas', 'sascalc', 'file_converter', 'core'))
-
-    import sas.sascalc.calculator
-    sas.sascalc.calculator.core = import_package('sas.sascalc.calculator.core',
-                                                 joinpath(build_path, 'sas', 'sascalc', 'calculator', 'core'))
-
-    sys.path.append(build_path)
 
     set_git_tag()

setup.py

@@ -12 +12 @@
 import shutil
 import sys
-from distutils.command.build_ext import build_ext
-from distutils.core import Command
 
 import numpy as np
+
 from setuptools import Extension, setup
+from setuptools import Command
+from setuptools.command.build_ext import build_ext
 
 try:
@@ -22 +23 @@
 except ImportError:
     pass
+
+# Convert "test" argument to "pytest" so 'python setup.py test' works
+sys.argv = [("pytest" if s == "test" else s) for s in sys.argv]
 
 # Manage version number ######################################
@@ -246 +250 @@
 # sas.sascalc.calculator
 gen_dir = os.path.join("src", "sas", "sascalc", "calculator", "c_extensions")
-package_dir["sas.sascalc.calculator.core"] = gen_dir
 package_dir["sas.sascalc.calculator"] = os.path.join(
     "src", "sas", "sascalc", "calculator")
-packages.extend(["sas.sascalc.calculator", "sas.sascalc.calculator.core"])
-ext_modules.append(Extension("sas.sascalc.calculator.core.sld2i",
+packages.append("sas.sascalc.calculator")
+ext_modules.append(Extension("sas.sascalc.calculator._sld2i",
                              sources=[
                                  os.path.join(gen_dir, "sld2i_module.c"),
@@ -258 +261 @@
                              ],
                              include_dirs=[gen_dir],
-                             )
-                   )
+                             ))
 
 # sas.sascalc.pr
 srcdir = os.path.join("src", "sas", "sascalc", "pr", "c_extensions")
-package_dir["sas.sascalc.pr.core"] = srcdir
 package_dir["sas.sascalc.pr"] = os.path.join("src", "sas", "sascalc", "pr")
-packages.extend(["sas.sascalc.pr", "sas.sascalc.pr.core"])
-ext_modules.append(Extension("sas.sascalc.pr.core.pr_inversion",
+packages.append("sas.sascalc.pr")
+ext_modules.append(Extension("sas.sascalc.pr._pr_inversion",
                              sources=[os.path.join(srcdir, "Cinvertor.c"),
                                       os.path.join(srcdir, "invertor.c"),
@@ -276 +277 @@
 # sas.sascalc.file_converter
 mydir = os.path.join("src", "sas", "sascalc", "file_converter", "c_ext")
-package_dir["sas.sascalc.file_converter.core"] = mydir
 package_dir["sas.sascalc.file_converter"] = os.path.join(
     "src", "sas", "sascalc", "file_converter")
-packages.extend(["sas.sascalc.file_converter",
-                 "sas.sascalc.file_converter.core"])
-ext_modules.append(Extension("sas.sascalc.file_converter.core.bsl_loader",
+packages.append("sas.sascalc.file_converter")
+ext_modules.append(Extension("sas.sascalc.file_converter._bsl_loader",
                              sources=[os.path.join(mydir, "bsl_loader.c")],
                              include_dirs=[np.get_include()],
@@ -443 +442 @@
     cmdclass={'build_ext': build_ext_subclass,
               'docs': BuildSphinxCommand,
-              'disable_openmp': DisableOpenMPCommand}
+              'disable_openmp': DisableOpenMPCommand},
+    setup_requires=['pytest-runner'] if 'pytest' in sys.argv else [],
+    tests_require=['pytest'],
 )

src/sas/sascalc/calculator/c_extensions/sld2i_module.c

@@ -2 +2 @@
   SLD2I module to perform point and I calculations
  */
+#include <stdio.h>
+
+//#define Py_LIMITED_API 0x03020000
 #include <Python.h>
-#include <stdio.h>
+
 #include "sld2i.h"
 
@@ -160 +163 @@
 
 #define MODULE_DOC "Sld2i C Library"
-#define MODULE_NAME "sld2i"
-#define MODULE_INIT2 initsld2i
-#define MODULE_INIT3 PyInit_sld2i
+#define MODULE_NAME "_sld2i"
+#define MODULE_INIT2 init_sld2i
+#define MODULE_INIT3 PyInit__sld2i
 #define MODULE_METHODS module_methods
 

src/sas/sascalc/calculator/sas_gen.py

@@ -14 +14 @@
 import numpy as np
 
-from .core import sld2i as mod
+from . import _sld2i
 from .BaseComponent import BaseComponent
 
@@ -145 +145 @@
             self.params['Up_frac_out'],
             self.params['Up_theta'])
-        model = mod.new_GenI(*args)
+        model = _sld2i.new_GenI(*args)
         if len(qy):
             qx, qy = _vec(qx), _vec(qy)
             I_out = np.empty_like(qx)
             #print("npoints", qx.shape, "npixels", pos_x.shape)
-            mod.genicomXY(model, qx, qy, I_out)
+            _sld2i.genicomXY(model, qx, qy, I_out)
             #print("I_out after", I_out)
         else:
             qx = _vec(qx)
             I_out = np.empty_like(qx)
-            mod.genicom(model, qx, I_out)
+            _sld2i.genicom(model, qx, I_out)
         vol_correction = self.data_total_volume / self.params['total_volume']
         result = (self.params['scale'] * vol_correction * I_out
@@ -304 +304 @@
                 z_dir2 *= z_dir2
                 mask = (x_dir2 + y_dir2 + z_dir2) <= 1.0
-            except Exception:
-                logger.error(sys.exc_value)
+            except Exception as exc:
+                logger.error(exc)
         self.output = MagSLD(self.pos_x[mask], self.pos_y[mask],
                              self.pos_z[mask], self.sld_n[mask],
@@ -600 +600 @@
                         y_lines.append(y_line)
                         z_lines.append(z_line)
-                except Exception:
-                    logger.error(sys.exc_value)
+                except Exception as exc:
+                    logger.error(exc)
 
             output = MagSLD(pos_x, pos_y, pos_z, sld_n, sld_mx, sld_my, sld_mz)
@@ -691 +691 @@
                             _vol_pix = float(toks[7])
                             vol_pix = np.append(vol_pix, _vol_pix)
-                        except Exception:
+                        except Exception as exc:
                             vol_pix = None
-                    except Exception:
+                    except Exception as exc:
                         # Skip non-data lines
-                        logger.error(sys.exc_value)
+                        logger.error(exc)
             output = MagSLD(pos_x, pos_y, pos_z, sld_n,
                             sld_mx, sld_my, sld_mz)

src/sas/sascalc/file_converter/bsl_loader.py

@@ -1 @@
-from sas.sascalc.file_converter.core.bsl_loader import CLoader
+from sas.sascalc.file_converter._bsl_loader import CLoader
 from sas.sascalc.dataloader.data_info import Data2D
 from copy import deepcopy
@@ -67 +67 @@
                     'swap_bytes': int(metadata[3])
                 }
-            except:
+            except Exception:
                 is_valid = False
                 err_msg = "Invalid metadata in header file for {}"

src/sas/sascalc/file_converter/c_ext/bsl_loader.c

@@ +1 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+//#define Py_LIMITED_API 0x03020000
 #include <Python.h>
+#include <structmember.h>
 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
 #include <numpy/arrayobject.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include "structmember.h"
+
 #include "bsl_loader.h"
 
@@ -292 +295 @@
 
 #define MODULE_DOC "C module for loading bsl."
-#define MODULE_NAME "bsl_loader"
-#define MODULE_INIT2 initbsl_loader
-#define MODULE_INIT3 PyInit_bsl_loader
+#define MODULE_NAME "_bsl_loader"
+#define MODULE_INIT2 init_bsl_loader
+#define MODULE_INIT3 PyInit__bsl_loader
 #define MODULE_METHODS module_methods
 

src/sas/sascalc/pr/c_extensions/Cinvertor.c

@@ -5 +5 @@
  *
  */
-#include <Python.h>
-#include "structmember.h"
 #include <stdio.h>
 #include <stdlib.h>
@@ -12 +10 @@
 #include <time.h>
 
+//#define Py_LIMITED_API 0x03050000
+#include <Python.h>
+#include <structmember.h>
+
 #include "invertor.h"
-
 
 /// Error object for raised exceptions
@@ -1121 +1122 @@
 
 #define MODULE_DOC "C extension module for inversion to P(r)."
-#define MODULE_NAME "pr_inversion"
-#define MODULE_INIT2 initpr_inversion
-#define MODULE_INIT3 PyInit_pr_inversion
+#define MODULE_NAME "_pr_inversion"
+#define MODULE_INIT2 init_pr_inversion
+#define MODULE_INIT3 PyInit__pr_inversion
 #define MODULE_METHODS module_methods
 

src/sas/sascalc/pr/invertor.py

@@ -17 +17 @@
 from numpy.linalg import lstsq
 from scipy import optimize
-from sas.sascalc.pr.core.pr_inversion import Cinvertor
+from sas.sascalc.pr._pr_inversion import Cinvertor
 
 logger = logging.getLogger(__name__)
@@ -71 +71 @@
         A[j][i] = (Fourier transformed base function for point j)
 
-    We them choose a number of r-points, n_r, to evaluate the second
+    We then choose a number of r-points, n_r, to evaluate the second
     derivative of P(r) at. This is used as our regularization term.
     For a vector r of length n_r, the following n_r rows are set to ::
@@ -144 +144 @@
         x, y, err, d_max, q_min, q_max and alpha
         """
-        if   name == 'x':
+        if name == 'x':
             if 0.0 in value:
                 msg = "Invertor: one of your q-values is zero. "
@@ -268 +268 @@
             A[i][j] = (Fourier transformed base function for point j)
 
-        We them choose a number of r-points, n_r, to evaluate the second
+        We then choose a number of r-points, n_r, to evaluate the second
         derivative of P(r) at. This is used as our regularization term.
         For a vector r of length n_r, the following n_r rows are set to ::
@@ -416 +416 @@
             A[i][j] = (Fourier transformed base function for point j)
 
-        We them choose a number of r-points, n_r, to evaluate the second
+        We then choose a number of r-points, n_r, to evaluate the second
         derivative of P(r) at. This is used as our regularization term.
         For a vector r of length n_r, the following n_r rows are set to ::
@@ -473 +473 @@
 
         # Perform the inversion (least square fit)
-        c, chi2, _, _ = lstsq(a, b)
+        c, chi2, _, _ = lstsq(a, b, rcond=None)
         # Sanity check
         try:
@@ -497 +497 @@
             cov = np.linalg.pinv(inv_cov)
             err = math.fabs(chi2 / float(npts - nfunc)) * cov
-        except:
+        except Exception as exc:
             # We were not able to estimate the errors
             # Return an empty error matrix
-            logger.error(sys.exc_value)
+            logger.error(exc)
 
         # Keep a copy of the last output
@@ -537 +537 @@
 
         """
-        from num_term import NTermEstimator
+        from .num_term import NTermEstimator
         estimator = NTermEstimator(self.clone())
         try:
             return estimator.num_terms(isquit_func)
-        except:
+        except Exception as exc:
             # If we fail, estimate alpha and return the default
             # number of terms
             best_alpha, _, _ = self.estimate_alpha(self.nfunc)
-            logger.warning("Invertor.estimate_numterms: %s" % sys.exc_value)
+            logger.warning("Invertor.estimate_numterms: %s" % exc)
             return self.nfunc, best_alpha, "Could not estimate number of terms"
 
@@ -631 +631 @@
                 return best_alpha, message, elapsed
 
-        except:
-            message = "Invertor.estimate_alpha: %s" % sys.exc_value
+        except Exception as exc:
+            message = "Invertor.estimate_alpha: %s" % exc
             return 0, message, elapsed
 
@@ -748 +748 @@
                         self.cov[i][i] = float(toks2[1])
 
-            except:
-                msg = "Invertor.from_file: corrupted file\n%s" % sys.exc_value
+            except Exception as exc:
+                msg = "Invertor.from_file: corrupted file\n%s" % exc
                 raise RuntimeError(msg)
         else: