Statistics
| Branch: | Tag: | Revision:

root / cloudcms / rstutils.py @ d240ebcb

History | View | Annotate | Download (5.8 kB)

1 deb708bf Kostas Papadimitriou
# -*- coding: utf-8 -*-
2 deb708bf Kostas Papadimitriou
3 deb708bf Kostas Papadimitriou
"""
4 deb708bf Kostas Papadimitriou
Helper methods to parse rst documents and extract data appropriate for faq/guide
5 deb708bf Kostas Papadimitriou
entries creation.
6 deb708bf Kostas Papadimitriou
"""
7 deb708bf Kostas Papadimitriou
8 deb708bf Kostas Papadimitriou
import os
9 deb708bf Kostas Papadimitriou
import sys
10 deb708bf Kostas Papadimitriou
import glob
11 deb708bf Kostas Papadimitriou
import StringIO
12 deb708bf Kostas Papadimitriou
13 deb708bf Kostas Papadimitriou
from os.path import join
14 deb708bf Kostas Papadimitriou
from collections import defaultdict
15 deb708bf Kostas Papadimitriou
from docutils.core import publish_parts
16 deb708bf Kostas Papadimitriou
from lxml import html
17 deb708bf Kostas Papadimitriou
18 deb708bf Kostas Papadimitriou
class SphinxImportException(Exception):
19 deb708bf Kostas Papadimitriou
    pass
20 deb708bf Kostas Papadimitriou
21 deb708bf Kostas Papadimitriou
22 deb708bf Kostas Papadimitriou
class SphinxImportValidationError(SphinxImportException):
23 deb708bf Kostas Papadimitriou
    pass
24 deb708bf Kostas Papadimitriou
25 deb708bf Kostas Papadimitriou
26 deb708bf Kostas Papadimitriou
def rst2html(data):
27 deb708bf Kostas Papadimitriou
    """
28 deb708bf Kostas Papadimitriou
    Use docutils publis_parts to convert rst to html. Return parts body and error
29 deb708bf Kostas Papadimitriou
    output tuple.
30 deb708bf Kostas Papadimitriou
    """
31 deb708bf Kostas Papadimitriou
    origstderr = sys.stderr
32 deb708bf Kostas Papadimitriou
    sys.stderr = StringIO.StringIO()
33 deb708bf Kostas Papadimitriou
34 deb708bf Kostas Papadimitriou
    parts = publish_parts(data, writer_name='html')['body']
35 deb708bf Kostas Papadimitriou
    sys.stderr.seek(0)
36 deb708bf Kostas Papadimitriou
    output = sys.stderr.read()
37 deb708bf Kostas Papadimitriou
    sys.stderr = origstderr
38 deb708bf Kostas Papadimitriou
39 deb708bf Kostas Papadimitriou
    return parts, output
40 deb708bf Kostas Papadimitriou
41 deb708bf Kostas Papadimitriou
42 deb708bf Kostas Papadimitriou
def parse_rst_data(data, data_type='faq'):
43 deb708bf Kostas Papadimitriou
    """
44 deb708bf Kostas Papadimitriou
    Parse given data from rst to html. Filter html and generate approriate
45 deb708bf Kostas Papadimitriou
    entries based on data_type provided.
46 deb708bf Kostas Papadimitriou

47 deb708bf Kostas Papadimitriou
    Generated content:
48 deb708bf Kostas Papadimitriou

49 deb708bf Kostas Papadimitriou
        - **category** (used for `faq` data type since each question belongs to a
50 deb708bf Kostas Papadimitriou
          specific category)
51 deb708bf Kostas Papadimitriou
        - **slug** the slug of the entry
52 deb708bf Kostas Papadimitriou
        - **title** the title of the entry
53 deb708bf Kostas Papadimitriou
        - **html_data** the html content of the entry
54 deb708bf Kostas Papadimitriou
        - **images** (img-alt, img-path) tuples list
55 deb708bf Kostas Papadimitriou
    """
56 deb708bf Kostas Papadimitriou
    html_data, output = rst2html(data)
57 deb708bf Kostas Papadimitriou
    doc = html.document_fromstring("<html><body>" + html_data + "</body></html>")
58 deb708bf Kostas Papadimitriou
59 deb708bf Kostas Papadimitriou
    category_selectors = {
60 deb708bf Kostas Papadimitriou
        'faq': ".//div[h2][@class='section']",
61 deb708bf Kostas Papadimitriou
        'userguide': ".//div[h1][@class='section']",
62 deb708bf Kostas Papadimitriou
    }
63 deb708bf Kostas Papadimitriou
64 deb708bf Kostas Papadimitriou
    # find first level sections
65 deb708bf Kostas Papadimitriou
    sections = doc.findall(category_selectors[data_type])
66 deb708bf Kostas Papadimitriou
    for section in sections:
67 deb708bf Kostas Papadimitriou
        entry_category = (None, None)
68 deb708bf Kostas Papadimitriou
69 deb708bf Kostas Papadimitriou
        attrs = dict(section.items())
70 deb708bf Kostas Papadimitriou
        if not attrs.get('id', None):
71 deb708bf Kostas Papadimitriou
            continue
72 deb708bf Kostas Papadimitriou
73 deb708bf Kostas Papadimitriou
        slug = attrs.get('id')
74 deb708bf Kostas Papadimitriou
        if data_type == 'userguide':
75 deb708bf Kostas Papadimitriou
            title = section.find('h1').text_content()
76 deb708bf Kostas Papadimitriou
            section.remove(section.find('h1'))
77 deb708bf Kostas Papadimitriou
        else:
78 deb708bf Kostas Papadimitriou
            title = section.find('h2').text_content()
79 deb708bf Kostas Papadimitriou
            section.remove(section.find('h2'))
80 deb708bf Kostas Papadimitriou
81 deb708bf Kostas Papadimitriou
        image_els = section.findall('.//img')
82 deb708bf Kostas Papadimitriou
83 deb708bf Kostas Papadimitriou
        if data_type == 'faq':
84 deb708bf Kostas Papadimitriou
            h1 = list(section.iterancestors())[0].find(".//h1")
85 deb708bf Kostas Papadimitriou
            el_with_id = dict(h1.getparent().items())
86 deb708bf Kostas Papadimitriou
            entry_category = (el_with_id.get('id', None), h1.text_content())
87 deb708bf Kostas Papadimitriou
88 deb708bf Kostas Papadimitriou
89 deb708bf Kostas Papadimitriou
        def get_img_el_data(img):
90 deb708bf Kostas Papadimitriou
            attrs = dict(img.items())
91 deb708bf Kostas Papadimitriou
            alt = attrs.get('alt', None)
92 deb708bf Kostas Papadimitriou
            if not alt:
93 deb708bf Kostas Papadimitriou
                alt = "okeanos iaas " + data_type + " image"
94 deb708bf Kostas Papadimitriou
            else:
95 deb708bf Kostas Papadimitriou
                if len(alt.split("/")) > 0:
96 deb708bf Kostas Papadimitriou
                    alt = data_type + " " + alt.split("/")[-1]
97 deb708bf Kostas Papadimitriou
                if len(alt.split(".")) > 0:
98 deb708bf Kostas Papadimitriou
                    alt = alt.split(".")[0]
99 deb708bf Kostas Papadimitriou
100 deb708bf Kostas Papadimitriou
            img.set('alt', alt)
101 deb708bf Kostas Papadimitriou
102 deb708bf Kostas Papadimitriou
            src = attrs.get('src')
103 deb708bf Kostas Papadimitriou
            if src.startswith("/images"):
104 deb708bf Kostas Papadimitriou
                src = src[1:]
105 deb708bf Kostas Papadimitriou
                img.set('src', src)
106 deb708bf Kostas Papadimitriou
107 deb708bf Kostas Papadimitriou
            return attrs.get('alt', None), src
108 deb708bf Kostas Papadimitriou
109 deb708bf Kostas Papadimitriou
        images = map(get_img_el_data, image_els)
110 deb708bf Kostas Papadimitriou
111 deb708bf Kostas Papadimitriou
        html_data = ""
112 deb708bf Kostas Papadimitriou
        for child in section.getchildren():
113 deb708bf Kostas Papadimitriou
            html_data += html.tostring(child, pretty_print=True)
114 deb708bf Kostas Papadimitriou
115 deb708bf Kostas Papadimitriou
        yield entry_category, slug, title, html_data, images, output
116 deb708bf Kostas Papadimitriou
117 deb708bf Kostas Papadimitriou
118 deb708bf Kostas Papadimitriou
def get_dir_rst_files(dirname):
119 deb708bf Kostas Papadimitriou
    """
120 deb708bf Kostas Papadimitriou
    Given a dir return the glob of *.rst files
121 deb708bf Kostas Papadimitriou
    """
122 deb708bf Kostas Papadimitriou
    for f in glob.glob(join(dirname, '*.rst')):
123 deb708bf Kostas Papadimitriou
        if f.startswith('index'):
124 deb708bf Kostas Papadimitriou
            continue
125 deb708bf Kostas Papadimitriou
        yield f
126 deb708bf Kostas Papadimitriou
127 deb708bf Kostas Papadimitriou
128 deb708bf Kostas Papadimitriou
def generate_rst_contents_from_dir(rstdir):
129 deb708bf Kostas Papadimitriou
    """
130 deb708bf Kostas Papadimitriou
    Handle directory contents and run ``parse_rst_data`` for each file we want
131 deb708bf Kostas Papadimitriou
    to parse.
132 deb708bf Kostas Papadimitriou

133 deb708bf Kostas Papadimitriou
    Valid structure of the dir contents so that appropriate files can be parsed::
134 deb708bf Kostas Papadimitriou

135 deb708bf Kostas Papadimitriou
        ├── README.rst
136 deb708bf Kostas Papadimitriou
        └── source
137 deb708bf Kostas Papadimitriou
            ├── conf.py
138 deb708bf Kostas Papadimitriou
            ├── faq
139 deb708bf Kostas Papadimitriou
            │   ├── cyclades.rst
140 deb708bf Kostas Papadimitriou
            │   ├── index.rst
141 deb708bf Kostas Papadimitriou
            │   ├── okeanos.rst
142 deb708bf Kostas Papadimitriou
            │   └── pithos.rst
143 deb708bf Kostas Papadimitriou
            ├── images
144 deb708bf Kostas Papadimitriou
            │   ├── cyclades
145 deb708bf Kostas Papadimitriou
            │   │   ├── image10.png
146 deb708bf Kostas Papadimitriou
            │   │   └── image9.png
147 deb708bf Kostas Papadimitriou
            │   ├── faq
148 deb708bf Kostas Papadimitriou
            │   │   └── faq_image1.png
149 deb708bf Kostas Papadimitriou
            │   ├── intro_img_cyclades.png
150 deb708bf Kostas Papadimitriou
            │   └── pithos_guide
151 deb708bf Kostas Papadimitriou
            │       └── image2.png
152 deb708bf Kostas Papadimitriou
            ├── index.rst
153 deb708bf Kostas Papadimitriou
            └── userguide
154 deb708bf Kostas Papadimitriou
                ├── cyclades.rst
155 deb708bf Kostas Papadimitriou
                ├── index.rst
156 deb708bf Kostas Papadimitriou
                ├── pithos.rst
157 deb708bf Kostas Papadimitriou
                └── quick-intro.rst
158 deb708bf Kostas Papadimitriou

159 deb708bf Kostas Papadimitriou
    Will generate a tuple of,
160 deb708bf Kostas Papadimitriou

161 deb708bf Kostas Papadimitriou
        ['faq', 'userguide'], </abs/path/filename.rst> + *<generated tuple members of ``parse_rst_data``>
162 deb708bf Kostas Papadimitriou

163 deb708bf Kostas Papadimitriou
    """
164 deb708bf Kostas Papadimitriou
165 deb708bf Kostas Papadimitriou
    #rstdir = "/tmp/tmphsl6bicloudcms-sphinx-exports"
166 deb708bf Kostas Papadimitriou
167 deb708bf Kostas Papadimitriou
    fpath = lambda x: join(rstdir, 'source', x)
168 deb708bf Kostas Papadimitriou
169 deb708bf Kostas Papadimitriou
    images_dir = fpath('images')
170 deb708bf Kostas Papadimitriou
    guide_dir = fpath('userguide')
171 deb708bf Kostas Papadimitriou
    faq_dir = fpath('faq')
172 deb708bf Kostas Papadimitriou
173 deb708bf Kostas Papadimitriou
    # validation
174 deb708bf Kostas Papadimitriou
    if not os.path.exists(images_dir) or not os.path.isdir(images_dir):
175 deb708bf Kostas Papadimitriou
        raise SphinxImportException('Cannot find images dir')
176 deb708bf Kostas Papadimitriou
177 deb708bf Kostas Papadimitriou
    if not os.path.exists(guide_dir) or not os.path.isdir(guide_dir):
178 deb708bf Kostas Papadimitriou
        raise SphinxImportException('Cannot find guide dir')
179 deb708bf Kostas Papadimitriou
180 deb708bf Kostas Papadimitriou
    if not os.path.exists(faq_dir) or not os.path.isdir(faq_dir):
181 deb708bf Kostas Papadimitriou
        raise SphinxImportException('Cannot find FAQs dir')
182 deb708bf Kostas Papadimitriou
183 deb708bf Kostas Papadimitriou
    def fix_image_path(img):
184 deb708bf Kostas Papadimitriou
        # make image path absolute
185 deb708bf Kostas Papadimitriou
        img = list(img)
186 deb708bf Kostas Papadimitriou
        if img[1].startswith("/"):
187 deb708bf Kostas Papadimitriou
            img.append(fpath(img[1][1:]))
188 deb708bf Kostas Papadimitriou
        else:
189 deb708bf Kostas Papadimitriou
            img.append(fpath(img[1]))
190 deb708bf Kostas Papadimitriou
191 deb708bf Kostas Papadimitriou
        return img
192 deb708bf Kostas Papadimitriou
193 deb708bf Kostas Papadimitriou
    for d in ['userguide', 'faq']:
194 deb708bf Kostas Papadimitriou
        for f in get_dir_rst_files(fpath(d)):
195 deb708bf Kostas Papadimitriou
            for category, slug, title, html_data, \
196 deb708bf Kostas Papadimitriou
                    images, stderr in parse_rst_data(file(f).read(), d):
197 deb708bf Kostas Papadimitriou
                # absolute image paths
198 deb708bf Kostas Papadimitriou
                images = map(fix_image_path, images)
199 deb708bf Kostas Papadimitriou
                yield d, f, category, slug, title, html_data, images, stderr
200 deb708bf Kostas Papadimitriou
201 deb708bf Kostas Papadimitriou