source: sasview/docs/sphinx-docs/build_sphinx.py @ 28c4a3d

ESS_GUIESS_GUI_DocsESS_GUI_batch_fittingESS_GUI_bumps_abstractionESS_GUI_iss1116ESS_GUI_iss879ESS_GUI_iss959ESS_GUI_openclESS_GUI_orderingESS_GUI_sync_sascalcmagnetic_scattrelease-4.2.2ticket-1009ticket-1094-headlessticket-1242-2d-resolutionticket-1243ticket-1249ticket885unittest-saveload
Last change on this file since 28c4a3d was 9d566b2, checked in by Paul Kienzle <pkienzle@…>, 7 years ago

katex support
  • Property mode set to 100755
File size: 18.8 KB
Line 
1#!/usr/bin/env python
2"""
3Functions for building sphinx docs.
4
5For more information on the invocation of sphinx see:
6http://sphinx-doc.org/invocation.html
7"""
8from __future__ import print_function
9
10import subprocess
11import os
12import sys
13import fnmatch
14import shutil
15import imp
16
17from glob import glob
18from distutils.dir_util import copy_tree
19from distutils.util import get_platform
20from shutil import copy
21from os import listdir
22
23platform = '.%s-%s'%(get_platform(), sys.version[:3])
24
25CURRENT_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
26
27run = imp.load_source('run', os.path.join(CURRENT_SCRIPT_DIR, '..', '..', 'run.py'))
28run.prepare()
29
30SASVIEW_SRC = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "src")
31SASVIEW_BUILD = os.path.abspath(os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "build", "lib"+platform))
32SASVIEW_DOCS = os.path.join(SASVIEW_BUILD, "doc")
33SASVIEW_TEST = os.path.join(SASVIEW_SRC, "..", "sasview", "test", "media")
34SASVIEW_TOC_SOURCE = os.path.join(CURRENT_SCRIPT_DIR, "source")
35
36# Need to slurp in the new sasmodels model definitions to replace the old model_functions.rst
37# We are currently here:
38#/sasview-local-trunk/docs/sphinx-docs/build_sphinx.py
39SASMODELS_SOURCE_PROLOG = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc")
40SASMODELS_SOURCE_GPU = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "ref", "gpu")
41SASMODELS_SOURCE_SESANS = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "ref", "sesans")
42SASMODELS_SOURCE_SESANSIMG = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "ref", "sesans", "sesans_img")
43SASMODELS_SOURCE_MAGNETISM = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "ref", "magnetism")
44SASMODELS_SOURCE_MAGIMG = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "ref", "magnetism", "mag_img")
45SASMODELS_SOURCE_REF_MODELS = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "ref", "models")
46SASMODELS_SOURCE_MODELS = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "model")
47SASMODELS_SOURCE_IMG = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "model", "img")
48SASMODELS_SOURCE_AUTOIMG = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "_build", "html", "_images")
49## Don't do assemble-in-place
50## Assemble the docs in a temporary folder
51SASMODELS_DEST_PROLOG = os.path.join(CURRENT_SCRIPT_DIR, "source-temp")
52SASMODELS_DEST_REF_MODELS = os.path.join(SASMODELS_DEST_PROLOG, "user")
53SASMODELS_DEST_MODELS = os.path.join(SASMODELS_DEST_PROLOG, "user", "models")
54SASMODELS_DEST_IMG = os.path.join(SASMODELS_DEST_PROLOG, "user", "model-imgs", "new-models")
55SASMODELS_DEST_MAGIMG = os.path.join(SASMODELS_DEST_PROLOG, "user", "mag_img")
56SASMODELS_DEST_SESANSIMG = os.path.join(SASMODELS_DEST_PROLOG, "user", "sesans_img")
57SASMODELS_DEST_BUILDIMG = os.path.join(SASMODELS_DEST_PROLOG, "user", "models", "img")
58
59
60SPHINX_BUILD = os.path.join(CURRENT_SCRIPT_DIR, "build")
61SPHINX_SOURCE = os.path.join(CURRENT_SCRIPT_DIR, "source-temp")
62SPHINX_SOURCE_API = os.path.join(SPHINX_SOURCE, "dev", "api")
63SPHINX_SOURCE_GUIFRAME = os.path.join(SPHINX_SOURCE, "user", "sasgui", "guiframe")
64SPHINX_SOURCE_MODELS = os.path.join(SPHINX_SOURCE, "user", "models")
65SPHINX_SOURCE_PERSPECTIVES = os.path.join(SPHINX_SOURCE, "user", "sasgui", "perspectives")
66SPHINX_SOURCE_TEST = os.path.join(SPHINX_SOURCE, "test")
67SPHINX_SOURCE_USER = os.path.join(SPHINX_SOURCE, "user")
68KATEX_PARENT = os.path.join(SPHINX_SOURCE, "_static")
69KATEX_PATH = os.path.join(KATEX_PARENT, "katex")
70KATEX_VERSION = "v0.7.1"  # https://github.com/khan/katex/releases
71
72BUMPS_DOCS = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..",
73                          "bumps", "doc", "guide")
74BUMPS_TARGET = os.path.join(SPHINX_SOURCE_PERSPECTIVES, "fitting")
75
76def inplace_change(filename, old_string, new_string):
77# Thanks to http://stackoverflow.com/questions/4128144/replace-string-within-file-contents
78    s = open(filename).read()
79    if old_string in s:
80        print('Changing "{old_string}" to "{new_string}"'.format(**locals()))
81        s = s.replace(old_string, new_string)
82        f = open(filename, 'w')
83        f.write(s)
84        f.flush()
85        f.close()
86    else:
87        print('No occurences of "{old_string}" found.'.format(**locals()))
88
89def _remove_dir(dir_path):
90    """Removes the given directory."""
91    if os.path.isdir(dir_path):
92        print("Removing \"%s\"... " % dir_path)
93        shutil.rmtree(dir_path)
94
95def clean():
96    """
97    Clean the sphinx build directory.
98    """
99    print("=== Cleaning Sphinx Build ===")
100    _remove_dir(SASVIEW_DOCS)
101    _remove_dir(SPHINX_BUILD)
102    _remove_dir(SPHINX_SOURCE)
103    _remove_dir(KATEX_PATH)
104    #_remove_dir(SPHINX_SOURCE_GUIFRAME)
105    #_remove_dir(SPHINX_SOURCE_MODELS)
106    #_remove_dir(SPHINX_SOURCE_PERSPECTIVES)
107    #_remove_dir(SPHINX_SOURCE_TEST)
108
109def setup_source_temp():
110    """
111    Copy the source toctrees to new folder for assembling the sphinx-docs
112    """
113    print("=== Copying Source toctrees ===")
114    if os.path.exists(SASVIEW_TOC_SOURCE):
115        print("Found docs folder at", SASVIEW_TOC_SOURCE)
116        shutil.copytree(SASVIEW_TOC_SOURCE, SPHINX_SOURCE)
117
118def retrieve_user_docs():
119    """
120    Copies across the contents of any media/ directories in src/, and puts them
121    in an appropriately named directory of docs/sphinx-docs/source/. For
122    example:
123
124        sas/../[MODULE]/media/dir/A.rst
125        sas/../[MODULE]/media/B.rst
126
127    gets copied to a new location:
128
129        docs/sphinx-docs/source/user/[MODULE]/dir/A.rst
130        docs/sphinx-docs/source/user/[MODULE]/B.rst
131
132    so that Sphinx may pick it up when generating the documentation.
133    """
134    print("=== Retrieve User Docs ===")
135
136    # Copy documentation files from their "source" to their "destination".
137    for root, dirnames, _ in os.walk(SASVIEW_SRC):
138        for dirname in fnmatch.filter(dirnames, 'media'):
139
140            docs = os.path.abspath(os.path.join(root, dirname))
141            print("Found docs folder at \"%s\"." % docs)
142
143            dest_dir_part = os.path.dirname(os.path.relpath(docs, SASVIEW_SRC))
144            if os.sep in dest_dir_part:
145                dest_dir_part = dest_dir_part[dest_dir_part.index(os.sep) + 1:]
146            dest_dir = os.path.join(SPHINX_SOURCE, "user", dest_dir_part)
147
148            copy_tree(docs, dest_dir)
149
150    # Now pickup testdata_help.rst
151    print("=== Including Test Data Docs ===")
152    if os.path.exists(SASVIEW_TEST):
153        print("Found docs folder at", SASVIEW_TEST)
154        shutil.copytree(SASVIEW_TEST, SPHINX_SOURCE_TEST)
155
156    print("=== And the Sasmodels Docs ===")
157    # Make sure we have the relevant images for the new sasmodels documentation
158    # First(!) we'll make a local reference copy for SasView
159    # (/new-models will be cleaned each build)
160    if os.path.exists(SASMODELS_SOURCE_IMG):
161        print("Found img folder SASMODELS_SOURCE_IMG at", SASMODELS_SOURCE_IMG)
162        if not os.path.exists(SASMODELS_DEST_IMG):
163            print("Missing docs folder SASMODELS_DEST_IMG at", SASMODELS_DEST_IMG)
164            os.makedirs(SASMODELS_DEST_IMG)
165            print("created SASMODELS_DEST_BUILDIMG at", SASMODELS_DEST_BUILDIMG)
166        else:
167            print("Found img folder SASMODELS_DEST_IMG at", SASMODELS_DEST_IMG)
168        print("Copying sasmodels model image files...")
169        for files in os.listdir(SASMODELS_SOURCE_IMG):
170            fromhere = os.path.join(SASMODELS_SOURCE_IMG, files)
171            tohere = os.path.join(SASMODELS_DEST_IMG, files)
172            shutil.copy(fromhere, tohere)
173    else:
174        print("no source directory", SASMODELS_SOURCE_IMG, "was found")
175
176    if os.path.exists(SASMODELS_SOURCE_AUTOIMG):
177        print("Found img folder SASMODELS_SOURCE_AUTOIMG at", SASMODELS_SOURCE_AUTOIMG)
178        if not os.path.exists(SASMODELS_DEST_IMG):
179            print("Missing docs folder SASMODELS_DEST_IMG at", SASMODELS_DEST_IMG)
180            os.makedirs(SASMODELS_DEST_BUILDIMG)
181            print("created SASMODELS_DEST_BUILDIMG at", SASMODELS_DEST_BUILDIMG)
182        print("Copying sasmodels model auto-generated image files...")
183        for files in os.listdir(SASMODELS_SOURCE_AUTOIMG):
184            fromhere = os.path.join(SASMODELS_SOURCE_AUTOIMG, files)
185            tohere = os.path.join(SASMODELS_DEST_IMG, files)
186            shutil.copy(fromhere, tohere)
187    else:
188        print("no source directory", SASMODELS_SOURCE_AUTOIMG, "was found")
189
190    # And the rst prolog with the unit substitutions
191    if os.path.exists(SASMODELS_SOURCE_PROLOG):
192        print("Found prolog folder SASMODELS_SOURCE_PROLOG at", SASMODELS_SOURCE_PROLOG)
193        if os.path.exists(SASMODELS_DEST_PROLOG):
194            print("Found docs folder SASMODELS_DEST_PROLOG at", SASMODELS_DEST_PROLOG)
195            print("Copying sasmodels rst_prolog file...")
196            for files in os.listdir(SASMODELS_SOURCE_PROLOG):
197                if files.startswith("rst"):
198                    fromhere = os.path.join(SASMODELS_SOURCE_PROLOG, files)
199                    tohere = os.path.join(SASMODELS_DEST_PROLOG, files)
200                    shutil.copy(fromhere, tohere)
201    else:
202        print("no source directory", SASMODELS_SOURCE_PROLOG, "was found")
203
204    if os.path.exists(SASMODELS_SOURCE_GPU):
205        print("Found docs folder SASMODELS_SOURCE_GPU at", SASMODELS_SOURCE_GPU)
206        if os.path.exists(SPHINX_SOURCE_USER):
207            print("Found docs folder SPHINX_SOURCE_USER at", SPHINX_SOURCE_USER)
208            print("Copying sasmodels gpu files...")
209            for files in os.listdir(SASMODELS_SOURCE_GPU):
210                if files.endswith(".rst"):
211                    fromhere = os.path.join(SASMODELS_SOURCE_GPU, files)
212                    tohere = os.path.join(SPHINX_SOURCE_USER, files)
213                    shutil.copy(fromhere, tohere)
214    else:
215        print("no source directory", SASMODELS_SOURCE_GPU, "was found")
216
217    if os.path.exists(SASMODELS_SOURCE_SESANS):
218        print("Found docs folder SASMODELS_SOURCE_SESANS at", SASMODELS_SOURCE_SESANS)
219        if os.path.exists(SPHINX_SOURCE_USER):
220            print("Found docs folder SPHINX_SOURCE_USER at", SPHINX_SOURCE_USER)
221            print("Copying sasmodels sesans files...")
222            for files in os.listdir(SASMODELS_SOURCE_SESANS):
223                if files.endswith(".rst"):
224                    fromhere = os.path.join(SASMODELS_SOURCE_SESANS, files)
225                    tohere = os.path.join(SPHINX_SOURCE_USER, files)
226                    shutil.copy(fromhere, tohere)
227    else:
228        print("no source directory", SASMODELS_SOURCE_SESANS, "was found")
229
230    if os.path.exists(SASMODELS_SOURCE_MAGNETISM):
231        print("Found docs folder SASMODELS_SOURCE_MAGNETISM at", SASMODELS_SOURCE_MAGNETISM)
232        if os.path.exists(SASMODELS_DEST_REF_MODELS):
233            print("Found docs folder SASMODELS_DEST_REF_MODELS at", SASMODELS_DEST_REF_MODELS)
234            print("Copying sasmodels model toctree files...")
235            for files in os.listdir(SASMODELS_SOURCE_MAGNETISM):
236                if files.endswith(".rst"):
237                    fromhere = os.path.join(SASMODELS_SOURCE_MAGNETISM, files)
238                    tohere = os.path.join(SASMODELS_DEST_REF_MODELS, files)
239                    shutil.copy(fromhere, tohere)
240    else:
241        print("no source directory", SASMODELS_SOURCE_MAGNETISM, "was found")
242
243    if os.path.exists(SASMODELS_SOURCE_MAGIMG):
244        print("Found img folder SASMODELS_SOURCE_MAGIMG   at", SASMODELS_SOURCE_MAGIMG)
245        if not os.path.exists(SASMODELS_DEST_MAGIMG):
246            print("Missing img folder SASMODELS_DEST_MAGIMG at", SASMODELS_DEST_MAGIMG)
247            os.makedirs(SASMODELS_DEST_MAGIMG)
248            print("created SASMODELS_DEST_MAGIMG at", SASMODELS_DEST_MAGIMG)
249        print("Copying sasmodels mag image files...")
250        for files in os.listdir(SASMODELS_SOURCE_MAGIMG):
251            fromhere = os.path.join(SASMODELS_SOURCE_MAGIMG, files)
252            tohere = os.path.join(SASMODELS_DEST_MAGIMG, files)
253            shutil.copy(fromhere, tohere)
254    else:
255        print("no source directory", SASMODELS_SOURCE_MAGIMG, "was found")
256
257    if os.path.exists(SASMODELS_SOURCE_SESANSIMG):
258        print("Found img folder SASMODELS_SOURCE_SESANSIMG at", SASMODELS_SOURCE_SESANSIMG)
259        if not os.path.exists(SASMODELS_DEST_SESANSIMG):
260            print("Missing img folder SASMODELS_DEST_SESANSIMG at", SASMODELS_DEST_SESANSIMG)
261            os.makedirs(SASMODELS_DEST_SESANSIMG)
262            print("created SASMODELS_DEST_SESANSIMG at", SASMODELS_DEST_SESANSIMG)
263        print("Copying sasmodels sesans image files...")
264        for files in os.listdir(SASMODELS_SOURCE_SESANSIMG):
265            fromhere = os.path.join(SASMODELS_SOURCE_SESANSIMG, files)
266            tohere = os.path.join(SASMODELS_DEST_SESANSIMG, files)
267            shutil.copy(fromhere, tohere)
268    else:
269        print("no source directory", SASMODELS_SOURCE_SESANSIMG, "was found")
270
271    if os.path.exists(SASMODELS_SOURCE_REF_MODELS):
272        print("Found docs folder SASMODELS_SOURCE_REF_MODELS at", SASMODELS_SOURCE_REF_MODELS)
273        if os.path.exists(SASMODELS_DEST_REF_MODELS):
274            print("Found docs folder SASMODELS_DEST_REF_MODELS at", SASMODELS_DEST_REF_MODELS)
275            print("Copying sasmodels model toctree files...")
276            for files in os.listdir(SASMODELS_SOURCE_REF_MODELS):
277                if files.endswith(".rst"):
278                    fromhere = os.path.join(SASMODELS_SOURCE_REF_MODELS, files)
279                    tohere = os.path.join(SASMODELS_DEST_REF_MODELS, files)
280                    shutil.copy(fromhere, tohere)
281            # But need to change the path to the model docs in the tocs
282            for files in os.listdir(SASMODELS_DEST_REF_MODELS):
283        #        print files
284                if files.startswith("shape"):
285                    print("Changing toc paths in", files)
286                    inplace_change(os.path.join(SASMODELS_DEST_REF_MODELS, files), "../../model/", "models/")
287                if files.startswith("sphere"):
288                    print("Changing toc paths in", files)
289                    inplace_change(os.path.join(SASMODELS_DEST_REF_MODELS, files), "../../model/", "models/")
290                if files.startswith("custom"):
291                    print("Changing toc paths in", files)
292                    inplace_change(os.path.join(SASMODELS_DEST_REF_MODELS, files), "../../model/", "models/")
293                if files.startswith("structure"):
294                    print("Changing toc paths in", files)
295                    inplace_change(os.path.join(SASMODELS_DEST_REF_MODELS, files), "../../model/", "models/")
296    else:
297        print("no source directory", SASMODELS_SOURCE_REF_MODELS, " was found")
298
299    if os.path.exists(SASMODELS_SOURCE_MODELS):
300        print("Found docs folder SASMODELS_SOURCE_MODELS at", SASMODELS_SOURCE_MODELS)
301        if os.path.exists(SASMODELS_DEST_MODELS):
302            print("Found docs folder SASMODELS_DEST_MODELS at", SASMODELS_DEST_MODELS)
303            print("Copying sasmodels model files...")
304            for files in os.listdir(SASMODELS_SOURCE_MODELS):
305                if files.endswith(".rst"):
306                    fromhere = os.path.join(SASMODELS_SOURCE_MODELS, files)
307                    tohere = os.path.join(SASMODELS_DEST_MODELS, files)
308                    shutil.copy(fromhere, tohere)
309        else:
310            print("Missing docs folder SASMODELS_DEST_MODELS at", SASMODELS_DEST_MODELS)
311            os.makedirs(SASMODELS_DEST_MODELS)
312            if not os.path.exists(SASMODELS_DEST_BUILDIMG):
313                os.makedirs(SASMODELS_DEST_BUILDIMG)
314            print("Created docs folder SASMODELS_DEST_MODELS at", SASMODELS_DEST_MODELS)
315            print("Copying model files for build...")
316            for files in os.listdir(SASMODELS_SOURCE_MODELS):
317                if files.endswith(".rst"):
318                    fromhere = os.path.join(SASMODELS_SOURCE_MODELS, files)
319                    tohere = os.path.join(SASMODELS_DEST_MODELS, files)
320                    shutil.copy(fromhere, tohere)
321            # No choice but to do this because model files are all coded for images in /models/img
322            print("Copying image files for build...")
323            for files in os.listdir(SASMODELS_DEST_IMG):
324                fromhere = os.path.join(SASMODELS_DEST_IMG, files)
325                tohere = os.path.join(SASMODELS_DEST_BUILDIMG, files)
326                shutil.copy(fromhere, tohere)
327    else:
328        print("no source directory", SASMODELS_SOURCE_MODELS, "was found.")
329        print("!!!!NO MODEL DOCS WILL BE BUILT!!!!")
330
331
332def fetch_katex(version, destination="_static"):
333    from zipfile import ZipFile
334    import urllib2
335    url = "https://github.com/Khan/KaTeX/releases/download/%s/katex.zip" % version
336    cache_path = "katex_%s.zip" % version
337    if not os.path.exists(cache_path):
338        try:
339            fd_in = urllib2.urlopen(url)
340            with open(cache_path, "wb") as fd_out:
341                fd_out.write(fd_in.read())
342        finally:
343            fd_in.close()
344    with ZipFile(cache_path) as zip:
345        zip.extractall(destination)
346
347
348def retrieve_bumps_docs():
349    """
350    Copies select files from the bumps documentation into fitting perspective
351    """
352    if os.path.exists(BUMPS_DOCS):
353        print("=== Retrieve BUMPS Docs ===")
354        filenames = [os.path.join(BUMPS_DOCS, "optimizer.rst")]
355        filenames += glob(os.path.join(BUMPS_DOCS, "dream-*.png"))
356        filenames += glob(os.path.join(BUMPS_DOCS, "fit-*.png"))
357        for f in filenames:
358            print("Copying file", f)
359            shutil.copy(f, BUMPS_TARGET)
360    else:
361        print("""
362======= Error =======
363missing directory %s
364The documentation will not include the optimizer selection section.
365Checkout the bumps source tree and rebuild the docs.
366""" % BUMPS_DOCS)
367
368def apidoc():
369    """
370    Runs sphinx-apidoc to generate .rst files from the docstrings in .py files
371    in the SasView build directory.
372    """
373    print("=== Generate API Rest Files ===")
374
375    # Clean directory before generating a new version.
376    _remove_dir(SPHINX_SOURCE_API)
377
378    subprocess.call(["sphinx-apidoc",
379                     "-o", SPHINX_SOURCE_API, # Output dir.
380                     "-d", "8", # Max depth of TOC.
381                     SASVIEW_BUILD])
382
383def build():
384    """
385    Runs sphinx-build.  Reads in all .rst files and spits out the final html.
386    """
387    print("=== Build HTML Docs from Rest Files ===")
388    subprocess.call(["sphinx-build",
389                     "-b", "html", # Builder name. TODO: accept as arg to setup.py.
390                     "-d", os.path.join(SPHINX_BUILD, "doctrees"),
391                     SPHINX_SOURCE,
392                     os.path.join(SPHINX_BUILD, "html")])
393
394    print("=== Copy HTML Docs to Build Directory ===")
395    html = os.path.join(SPHINX_BUILD, "html")
396    copy_tree(html, SASVIEW_DOCS)
397
398def rebuild():
399    clean()
400    setup_source_temp()
401    retrieve_user_docs()
402    retrieve_bumps_docs()
403    fetch_katex(version=KATEX_VERSION, destination=KATEX_PARENT)
404    apidoc()
405    build()
406
407    print("=== Done ===")
408
409if __name__ == "__main__":
410    rebuild()
Note: See TracBrowser for help on using the repository browser.