Initial import of the doc tools.

Author: birkenfeld

Makefile

@@ -0,0 +1,24 @@
+PYTHON ?= python
+
+export PYTHONPATH = $(shell echo "$$PYTHONPATH"):./sphinx
+
+.PHONY: all check clean clean-pyc pylint reindent testserver
+
+all: clean-pyc check
+
+check:
+	@$(PYTHON) utils/check_sources.py -i sphinx/style/jquery.js sphinx
+	@$(PYTHON) utils/check_sources.py converter
+
+clean: clean-pyc
+
+clean-pyc:
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+
+pylint:
+	@pylint --rcfile utils/pylintrc sphinx converter
+
+reindent:
+	@$(PYTHON) utils/reindent.py -r -B .

README

@@ -0,0 +1,79 @@
+py-rest-doc
+===========
+
+This sandbox project is about moving the official Python documentation
+to reStructuredText.
+
+
+What you need to know
+---------------------
+
+This project uses Python 2.5 features, so you'll need a working Python
+2.5 setup.
+
+If you want code highlighting, you need Pygments >= 0.8, easily
+installable from PyPI.  Jinja, the template engine, is included as a
+SVN external.
+
+For the rest of this document, let's assume that you have a Python
+checkout (you need the 2.6 line, i.e. the trunk) in ~/devel/python and
+this checkout in the current directory.
+
+To convert the LaTeX doc to reST, you first have to apply the patch in
+``etc/inst.diff`` to the ``inst/inst.tex`` LaTeX file in the Python
+checkout::
+
+   patch -d ~/devel/python/Doc -p0 < etc/inst.diff
+
+Then, create a target directory for the reST sources and run the
+converter script::
+
+   mkdir sources
+   python convert.py ~/devel/python/Doc sources
+
+This will convert all LaTeX sources to reST files in the ``sources``
+directory.
+
+The ``sources`` directory contains a ``conf.py`` file which contains
+general configuration for the build process, such as the Python
+version that should be shown, or the date format for "last updated on"
+notes.
+
+
+Building the HTML version
+-------------------------
+
+Then, create a target directory and run ::
+
+   mkdir build-html
+   python sphinx-build.py -b html sources build-html
+
+This will create HTML files in the ``build-html`` directory.
+
+The ``build-html`` directory will also contain a ``.doctrees``
+directory, which caches pickles containing the docutils doctrees for
+all source files, as well as an ``environment.pickle`` file that
+collects all meta-information and data that's needed to
+cross-reference the sources and generate indices.
+
+
+Running the online (web) version
+--------------------------------
+
+First, you need to build the source with the "web" builder::
+
+   mkdir build-web
+   python sphinx-build.py -b web sources build-web
+
+This will create files with pickled contents for the web application
+in the target directory.
+
+Then, you can run ::
+
+   python sphinx-web.py build-web
+
+which will start a webserver using wsgiref on ``localhost:3000``.  The
+web application has a configuration file ``build-web/webconf.py``,
+where you can configure the server and port for the application as
+well as different other settings specific to the web app.
+

TODO

@@ -0,0 +1,13 @@
+Global TODO
+===========
+
+- discuss and debug comments system
+- write new Makefile, handle automatic version info and checkout
+- write a "printable" builder (export to latex, most probably)
+- discuss the default role
+- discuss lib -> ref section move
+- prepare for databases other than sqlite for comments
+- look at the old tools/ scripts, what functionality should be rewritten
+- add search via Xapian?
+- optionally have a contents tree view in the sidebar (AJAX based)?
+

convert.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+"""
+    Convert the Python documentation to Sphinx
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: 2007 by Georg Brandl.
+    :license: Python license.
+"""
+
+import sys
+import os
+
+from converter import convert_dir
+
+if __name__ == '__main__':
+    try:
+        rootdir = sys.argv[1]
+        destdir = os.path.abspath(sys.argv[2])
+    except IndexError:
+        print "usage: convert.py docrootdir destdir"
+        sys.exit()
+
+    assert os.path.isdir(os.path.join(rootdir, 'texinputs'))
+    os.chdir(rootdir)
+    convert_dir(destdir, *sys.argv[3:])

converter/__init__.py

@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+"""
+    Documentation converter - high level functions
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: 2007 by Georg Brandl.
+    :license: Python license.
+"""
+
+import sys
+import os
+import glob
+import shutil
+import codecs
+from os import path
+
+from .tokenizer import Tokenizer
+from .latexparser import DocParser
+from .restwriter import RestWriter
+from .filenamemap import (fn_mapping, copyfiles_mapping, newfiles_mapping,
+                          rename_mapping, dirs_to_make, toctree_mapping,
+                          amendments_mapping)
+from .console import red, green
+
+def convert_file(infile, outfile, doraise=True, splitchap=False,
+                 toctree=None, deflang=None, labelprefix=''):
+    inf = codecs.open(infile, 'r', 'latin1')
+    p = DocParser(Tokenizer(inf.read()).tokenize(), infile)
+    if not splitchap:
+        outf = codecs.open(outfile, 'w', 'utf-8')
+    else:
+        outf = None
+    r = RestWriter(outf, splitchap, toctree, deflang, labelprefix)
+    try:
+        r.write_document(p.parse())
+        if splitchap:
+            for i, chapter in enumerate(r.chapters[1:]):
+                coutf = codecs.open('%s/%d_%s' % (
+                    path.dirname(outfile), i+1, path.basename(outfile)),
+                                    'w', 'utf-8')
+                coutf.write(chapter.getvalue())
+                coutf.close()
+        else:
+            outf.close()
+        return 1, r.warnings
+    except Exception, err:
+        if doraise:
+            raise
+        return 0, str(err)
+
+
+def convert_dir(outdirname, *args):
+    # make directories
+    for dirname in dirs_to_make:
+        try:
+            os.mkdir(path.join(outdirname, dirname))
+        except OSError:
+            pass
+
+    # copy files (currently only non-tex includes)
+    for oldfn, newfn in copyfiles_mapping.iteritems():
+        newpathfn = path.join(outdirname, newfn)
+        globfns = glob.glob(oldfn)
+        if len(globfns) == 1 and not path.isdir(newpathfn):
+            shutil.copyfile(globfns[0], newpathfn)
+        else:
+            for globfn in globfns:
+                shutil.copyfile(globfn, path.join(newpathfn,
+                                                  path.basename(globfn)))
+
+    # convert tex files
+    # "doc" is not converted. It must be rewritten anyway.
+    for subdir in ('api', 'dist', 'ext', 'inst', 'commontex',
+                   'lib', 'mac', 'ref', 'tut', 'whatsnew'):
+        if args and subdir not in args:
+            continue
+        if subdir not in fn_mapping:
+            continue
+        newsubdir = fn_mapping[subdir]['__newname__']
+        deflang = fn_mapping[subdir].get('__defaulthighlightlang__')
+        labelprefix = fn_mapping[subdir].get('__labelprefix__', '')
+        for filename in sorted(os.listdir(subdir)):
+            if not filename.endswith('.tex'):
+                continue
+            filename = filename[:-4] # strip extension
+            newname = fn_mapping[subdir][filename]
+            if newname is None:
+                continue
+            if newname.endswith(':split'):
+                newname = newname[:-6]
+                splitchap = True
+            else:
+                splitchap = False
+            if '/' not in newname:
+                outfilename = path.join(outdirname, newsubdir, newname + '.rst')
+            else:
+                outfilename = path.join(outdirname, newname + '.rst')
+            toctree = toctree_mapping.get(path.join(subdir, filename))
+            infilename = path.join(subdir, filename + '.tex')
+            print green(infilename),
+            success, state = convert_file(infilename, outfilename, False,
+                                          splitchap, toctree, deflang, labelprefix)
+            if not success:
+                print red("ERROR:")
+                print red("    " + state)
+            else:
+                if state:
+                    print "warnings:"
+                    for warning in state:
+                        print "    " + warning
+
+    # rename files, e.g. splitted ones
+    for oldfn, newfn in rename_mapping.iteritems():
+        try:
+            if newfn is None:
+                os.unlink(path.join(outdirname, oldfn))
+            else:
+                os.rename(path.join(outdirname, oldfn),
+                          path.join(outdirname, newfn))
+        except OSError, err:
+            if err.errno == 2:
+                continue
+            raise
+
+    # copy new files
+    srcdirname = path.join(path.dirname(__file__), 'newfiles')
+    for fn, newfn in newfiles_mapping.iteritems():
+        shutil.copyfile(path.join(srcdirname, fn),
+                        path.join(outdirname, newfn))
+
+    # make amendments
+    for newfn, (pre, post) in amendments_mapping.iteritems():
+        fn = path.join(outdirname, newfn)
+        try:
+            ft = open(fn).read()
+        except Exception, err:
+            print "Error making amendments to %s: %s" % (newfn, err)
+            continue
+        else:
+            fw = open(fn, 'w')
+            fw.write(pre)
+            fw.write(ft)
+            fw.write(post)
+            fw.close()

converter/console.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+    Console utils
+    ~~~~~~~~~~~~~
+
+    Format colored console output.
+
+    :copyright: 1998-2004 by the Gentoo Foundation.
+    :copyright: 2006-2007 by Georg Brandl.
+    :license: GNU GPL.
+"""
+
+esc_seq = "\x1b["
+
+codes = {}
+codes["reset"]     = esc_seq + "39;49;00m"
+
+codes["bold"]      = esc_seq + "01m"
+codes["faint"]     = esc_seq + "02m"
+codes["standout"]  = esc_seq + "03m"
+codes["underline"] = esc_seq + "04m"
+codes["blink"]     = esc_seq + "05m"
+codes["overline"]  = esc_seq + "06m"  # Who made this up? Seriously.
+
+ansi_color_codes = []
+for x in xrange(30, 38):
+    ansi_color_codes.append("%im" % x)
+    ansi_color_codes.append("%i;01m" % x)
+
+rgb_ansi_colors = [
+    '0x000000', '0x555555', '0xAA0000', '0xFF5555',
+    '0x00AA00', '0x55FF55', '0xAA5500', '0xFFFF55',
+    '0x0000AA', '0x5555FF', '0xAA00AA', '0xFF55FF',
+    '0x00AAAA', '0x55FFFF', '0xAAAAAA', '0xFFFFFF'
+]
+
+for x in xrange(len(rgb_ansi_colors)):
+    codes[rgb_ansi_colors[x]] = esc_seq + ansi_color_codes[x]
+
+del x
+
+codes["black"]     = codes["0x000000"]
+codes["darkgray"]  = codes["0x555555"]
+
+codes["red"]       = codes["0xFF5555"]
+codes["darkred"]   = codes["0xAA0000"]
+
+codes["green"]     = codes["0x55FF55"]
+codes["darkgreen"] = codes["0x00AA00"]
+
+codes["yellow"]    = codes["0xFFFF55"]
+codes["brown"]     = codes["0xAA5500"]
+
+codes["blue"]      = codes["0x5555FF"]
+codes["darkblue"]  = codes["0x0000AA"]
+
+codes["fuchsia"]   = codes["0xFF55FF"]
+codes["purple"]    = codes["0xAA00AA"]
+
+codes["teal"]      = codes["0x00AAAA"]
+codes["turquoise"] = codes["0x55FFFF"]
+
+codes["white"]     = codes["0xFFFFFF"]
+codes["lightgray"] = codes["0xAAAAAA"]
+
+codes["darkteal"]   = codes["turquoise"]
+codes["darkyellow"] = codes["brown"]
+codes["fuscia"]     = codes["fuchsia"]
+codes["white"]      = codes["bold"]
+
+def nocolor():
+    "turn off colorization"
+    for code in codes:
+        codes[code] = ""
+
+def reset_color():
+    return codes["reset"]
+
+def colorize(color_key, text):
+    return codes[color_key] + text + codes["reset"]
+
+functions_colors = [
+    "bold", "white", "teal", "turquoise", "darkteal",
+    "fuscia", "fuchsia", "purple", "blue", "darkblue",
+    "green", "darkgreen", "yellow", "brown",
+    "darkyellow", "red", "darkred"
+]
+
+def create_color_func(color_key):
+    """
+    Return a function that formats its argument in the given color.
+    """
+    def derived_func(text):
+        return colorize(color_key, text)
+    return derived_func
+
+ns = locals()
+for c in functions_colors:
+    ns[c] = create_color_func(c)
+
+del c, ns