from django.contrib import admin
+from django.conf.urls.defaults import patterns
+from django.views.generic.simple import direct_to_template
+from django.http import HttpResponse
+from django.shortcuts import redirect
+from django.contrib import messages
+
+
from feincms.translations import admin_translationinline, short_language_code
from cloudcms import models
admin.site.register(models.Client, ClientAdmin)
admin.site.register(models.Service, ServiceAdmin)
+
+from cloudcms.forms import RstZipImportForm
+
+def import_from_sphinx(request):
+ if not request.user.is_superuser:
+ return HttpResponse(status=401)
+
+ context = {}
+ form = RstZipImportForm()
+
+ if request.method == 'POST':
+ form = RstZipImportForm(request.POST, request.FILES)
+ if form.is_valid():
+ try:
+ ret = form.save(request.user)
+ messages.add_message(request, messages.INFO, 'Form saved')
+ return redirect('/cmsmanage/sphinximport/')
+ except Exception, e:
+ context['exception'] = e
+
+ else:
+ context['error'] = True
+
+ context['form'] = form
+
+ return direct_to_template(request, 'cms/admin_import_guide_faq.html', context)
+
+sphinx_import = admin.site.admin_view(import_from_sphinx)
+
--- /dev/null
+# -*- coding: utf-8 -*-
+
+import zipfile
+import tempfile
+import os
+import glob
+import logging
+
+from django import forms
+from django.conf import settings
+from django.db import transaction
+from django.core.files import File
+
+from feincms.module.medialibrary.models import Category as MediaCategory, \
+ MediaFile
+
+from cloudcmsguide.models import UserGuideEntry
+from cloudcmsfaq.models import Category as FaqCategory, Question
+from cloudcms.rstutils import generate_rst_contents_from_dir
+from cloudcms.models import Service, ServiceTranslation, Application
+from feincms.content.raw.models import RawContent
+
+logger = logging.getLogger('cloudcms.rstimport')
+
+# base filename to service slug map
+DEFAULT_SERVICE_MAP = {
+ 'cyclades':'cyclades',
+ 'okeanos':'okeanos',
+ 'pithos':'pithos'
+}
+
+DEFAULT_REGION_MAP = {
+ 'faq':'main',
+ 'userguide':'main',
+}
+
+DEFAULT_RESIZE_GEOMETRY = (400, 400)
+
+SERVICE_MAP = getattr(settings, 'CMS_RST_IMPORT_SERVICE_FILE_MAP',
+ DEFAULT_SERVICE_MAP)
+REGIONS_MAP = getattr(settings, 'CMS_RST_IMPORT_REGIONS_MAP',
+ DEFAULT_REGION_MAP)
+RESIZE_GEOMETRY = getattr(settings, 'CMS_RST_IMPORT_RESIZE_GEOMETRY',
+ DEFAULT_RESIZE_GEOMETRY)
+
+def service_from_filename(rst):
+ fname = os.path.basename(rst).replace(".rst","")
+ service_slug = DEFAULT_SERVICE_MAP.get(fname, None)
+ if not service_slug:
+ return None
+
+ try:
+ return Service.objects.filter(translations__slug=service_slug)[0]
+ except IndexError:
+ return None
+ except Service.DoesNotExist:
+ return None
+
+ return None
+
+
+def get_media_category(slug):
+ return MediaCategory.objects.get(slug=slug)
+
+
+CATEGORIES_CHOICES = (('faqs', 'FAQs'), ('guide', 'User guide'))
+
+class RstZipImportForm(forms.Form):
+
+ FAQ_MEDIA_CATEGORY = 'faq-images'
+ GUIDE_MEDIA_CATEGORY = 'user-guide-images'
+
+ clean_data = forms.BooleanField(initial=False, required=False)
+ dry_run = forms.BooleanField(initial=True, required=False,
+ widget=forms.HiddenInput)
+ import_file = forms.FileField(required=True)
+
+ def __init__(self, *args, **kwargs):
+ super(RstZipImportForm, self).__init__(*args, **kwargs)
+ self.log_data = ""
+
+ def log(self, msg):
+ self.log_data += "\n" + msg
+
+ def get_tmp_file(self, f):
+ tmp = tempfile.mktemp('cloudcms-sphinx-import')
+ f.file.reset()
+ fd = file(tmp, 'w')
+ fd.write(f.read())
+ fd.close()
+ return tmp
+
+ def clean_import_file(self):
+ f = self.cleaned_data['import_file']
+ tmpfile = self.get_tmp_file(f)
+ if not zipfile.is_zipfile(tmpfile):
+ raise forms.ValidationError("Invalid zip file")
+ return f
+
+ def clean(self, *args, **kwargs):
+ data = super(RstZipImportForm, self).clean(*args, **kwargs)
+ return data
+
+ def clean_existing_data(self):
+ Question.objects.all().delete()
+ UserGuideEntry.objects.all().delete()
+ MediaFile.objects.filter(categories__slug__in=[self.FAQ_MEDIA_CATEGORY, \
+ self.GUIDE_MEDIA_CATEGORY]).delete()
+
+ def save(self, user, use_dir=None):
+ dry_run = self.cleaned_data.get('dry_run')
+ clean_data = self.cleaned_data.get('clean_data')
+ import_file = self.cleaned_data.get('import_file')
+
+ if not use_dir:
+ zipdir = tempfile.mkdtemp('cloudcms-sphinx-exports')
+ zipdatafile = self.get_tmp_file(import_file)
+ zipf = zipfile.ZipFile(file(zipdatafile))
+ zipf.extractall(zipdir)
+ else:
+ zipdir = use_dir
+
+ subdirs = os.listdir(zipdir)
+ if len(subdirs) == 1 and os.path.isdir(os.path.join(zipdir, \
+ subdirs[0])) and subdirs[0] != 'source':
+ zipdir = os.path.join(zipdir, subdirs[0])
+
+ #sid = transaction.savepoint()
+
+ if clean_data:
+ try:
+ self.clean_existing_data()
+ except Exception, e:
+ return False
+
+ ret = ""
+ try:
+ for data_type, rst, category, slug, title, html_content, \
+ images, stderr in generate_rst_contents_from_dir(zipdir):
+
+ ret += stderr
+ #logger.info("Parsed %s" % (rst, ))
+ if stderr:
+ pass
+ #logger.info("Error output: %s" % (stderr, ))
+
+ service = service_from_filename(rst)
+ if not service:
+ logger.info("Skipping entry for file '%s'. No category found" % rst)
+ continue
+
+
+ # first save media files
+ newimages = []
+ if data_type == 'userguide':
+ cat = get_media_category(self.GUIDE_MEDIA_CATEGORY)
+ if data_type == 'faq':
+ cat = get_media_category(self.FAQ_MEDIA_CATEGORY)
+
+ if not cat:
+ continue
+
+ for imgname, imgpath, imgabspath in images:
+ newalt, newpath = create_or_update_media_file(cat, \
+ imgname, imgabspath)
+
+ html_content = html_content.replace(imgpath, newpath)
+
+ # now html contains correct image links, we are ready to save
+ # the faq/guide content
+ if data_type == 'faq':
+ cat = add_or_update_faq_category(category[0], category[1])
+ question = add_or_update_faq_question(user, service, cat, slug, \
+ title, html_content)
+
+ if data_type == 'userguide':
+ guide_entry = add_or_update_guide_entry(user, service, slug, \
+ title, html_content)
+
+
+ except Exception, e:
+ logger.exception("RST import failed")
+ #transaction.savepoint_rollback(sid)
+ return False
+
+ if dry_run:
+ pass
+ #transaction.savepoint_rollback(sid)
+ else:
+ pass
+ #transaction.savepoint_commit(sid)
+
+
+def create_or_update_media_file(category, name, path):
+ name = category.title + " " + name
+
+ try:
+ # TODO: Check language ?????
+ mf = MediaFile.objects.get(categories=category, translations__caption=name)
+ except MediaFile.DoesNotExist:
+ mf = MediaFile()
+ mf.category = category
+ mf.file = File(open(path))
+ mf.save()
+ mf.translations.create(caption=name)
+
+ # TODO: Check language ?????
+ return mf.translations.all()[0].caption, mf.get_absolute_url()
+
+
+def add_or_update_faq_category(slug, name):
+ try:
+ category = FaqCategory.objects.get(translations__slug=slug)
+ except FaqCategory.DoesNotExist:
+ category = FaqCategory()
+ category.save()
+ category.translations.create(slug=slug, title=name)
+
+ return category
+
+def add_or_update_faq_question(author, service, category, slug,\
+ title, html_content):
+
+ try:
+ q = Question.objects.get(slug=slug)
+ except:
+ q = Question()
+
+ q.author = author
+ q.is_active = True
+ q.category = category
+ q.service = service
+ q.slug = slug
+ q.title = title
+ q.save()
+ q.application = [Application.current()]
+ q.save()
+
+ RawContentModel = Question.content_type_for(RawContent)
+ try:
+ content = q.rawcontent_set.filter()[0]
+ except:
+ content = q.rawcontent_set.create(region=REGIONS_MAP['faq'])
+
+ content.text = html_content
+ content.save()
+
+ return q
+
+def add_or_update_guide_entry(author, service, slug, title, html_content):
+ try:
+ guide = UserGuideEntry.objects.get(slug=slug)
+ except:
+ guide = UserGuideEntry()
+
+ guide.author = author
+ guide.is_active = True
+ guide.service = service
+ guide.slug = slug
+ guide.title = title
+ guide.save()
+
+ RawContentModel = UserGuideEntry.content_type_for(RawContent)
+ try:
+ content = guide.rawcontent_set.filter()[0]
+ except:
+ content = guide.rawcontent_set.create(region=REGIONS_MAP['userguide'])
+
+ content.text = html_content
+ content.save()
+
+ return guide
+
--- /dev/null
+# -*- coding: utf-8 -*-
+
+"""
+Helper methods to parse rst documents and extract data appropriate for faq/guide
+entries creation.
+"""
+
+import os
+import sys
+import glob
+import StringIO
+
+from os.path import join
+from collections import defaultdict
+from docutils.core import publish_parts
+from lxml import html
+
+class SphinxImportException(Exception):
+ pass
+
+
+class SphinxImportValidationError(SphinxImportException):
+ pass
+
+
+def rst2html(data):
+ """
+ Use docutils publis_parts to convert rst to html. Return parts body and error
+ output tuple.
+ """
+ origstderr = sys.stderr
+ sys.stderr = StringIO.StringIO()
+
+ parts = publish_parts(data, writer_name='html')['body']
+ sys.stderr.seek(0)
+ output = sys.stderr.read()
+ sys.stderr = origstderr
+
+ return parts, output
+
+
+def parse_rst_data(data, data_type='faq'):
+ """
+ Parse given data from rst to html. Filter html and generate approriate
+ entries based on data_type provided.
+
+ Generated content:
+
+ - **category** (used for `faq` data type since each question belongs to a
+ specific category)
+ - **slug** the slug of the entry
+ - **title** the title of the entry
+ - **html_data** the html content of the entry
+ - **images** (img-alt, img-path) tuples list
+ """
+ html_data, output = rst2html(data)
+ doc = html.document_fromstring("<html><body>" + html_data + "</body></html>")
+
+ category_selectors = {
+ 'faq': ".//div[h2][@class='section']",
+ 'userguide': ".//div[h1][@class='section']",
+ }
+
+ # find first level sections
+ sections = doc.findall(category_selectors[data_type])
+ for section in sections:
+ entry_category = (None, None)
+
+ attrs = dict(section.items())
+ if not attrs.get('id', None):
+ continue
+
+ slug = attrs.get('id')
+ if data_type == 'userguide':
+ title = section.find('h1').text_content()
+ section.remove(section.find('h1'))
+ else:
+ title = section.find('h2').text_content()
+ section.remove(section.find('h2'))
+
+ image_els = section.findall('.//img')
+
+ if data_type == 'faq':
+ h1 = list(section.iterancestors())[0].find(".//h1")
+ el_with_id = dict(h1.getparent().items())
+ entry_category = (el_with_id.get('id', None), h1.text_content())
+
+
+ def get_img_el_data(img):
+ attrs = dict(img.items())
+ alt = attrs.get('alt', None)
+ if not alt:
+ alt = "okeanos iaas " + data_type + " image"
+ else:
+ if len(alt.split("/")) > 0:
+ alt = data_type + " " + alt.split("/")[-1]
+ if len(alt.split(".")) > 0:
+ alt = alt.split(".")[0]
+
+ img.set('alt', alt)
+
+ src = attrs.get('src')
+ if src.startswith("/images"):
+ src = src[1:]
+ img.set('src', src)
+
+ return attrs.get('alt', None), src
+
+ images = map(get_img_el_data, image_els)
+
+ html_data = ""
+ for child in section.getchildren():
+ html_data += html.tostring(child, pretty_print=True)
+
+ yield entry_category, slug, title, html_data, images, output
+
+
+def get_dir_rst_files(dirname):
+ """
+ Given a dir return the glob of *.rst files
+ """
+ for f in glob.glob(join(dirname, '*.rst')):
+ if f.startswith('index'):
+ continue
+ yield f
+
+
+def generate_rst_contents_from_dir(rstdir):
+ """
+ Handle directory contents and run ``parse_rst_data`` for each file we want
+ to parse.
+
+ Valid structure of the dir contents so that appropriate files can be parsed::
+
+ ├── README.rst
+ └── source
+ ├── conf.py
+ ├── faq
+ │ ├── cyclades.rst
+ │ ├── index.rst
+ │ ├── okeanos.rst
+ │ └── pithos.rst
+ ├── images
+ │ ├── cyclades
+ │ │ ├── image10.png
+ │ │ └── image9.png
+ │ ├── faq
+ │ │ └── faq_image1.png
+ │ ├── intro_img_cyclades.png
+ │ └── pithos_guide
+ │ └── image2.png
+ ├── index.rst
+ └── userguide
+ ├── cyclades.rst
+ ├── index.rst
+ ├── pithos.rst
+ └── quick-intro.rst
+
+ Will generate a tuple of,
+
+ ['faq', 'userguide'], </abs/path/filename.rst> + *<generated tuple members of ``parse_rst_data``>
+
+ """
+
+ #rstdir = "/tmp/tmphsl6bicloudcms-sphinx-exports"
+
+ fpath = lambda x: join(rstdir, 'source', x)
+
+ images_dir = fpath('images')
+ guide_dir = fpath('userguide')
+ faq_dir = fpath('faq')
+
+ # validation
+ if not os.path.exists(images_dir) or not os.path.isdir(images_dir):
+ raise SphinxImportException('Cannot find images dir')
+
+ if not os.path.exists(guide_dir) or not os.path.isdir(guide_dir):
+ raise SphinxImportException('Cannot find guide dir')
+
+ if not os.path.exists(faq_dir) or not os.path.isdir(faq_dir):
+ raise SphinxImportException('Cannot find FAQs dir')
+
+ def fix_image_path(img):
+ # make image path absolute
+ img = list(img)
+ if img[1].startswith("/"):
+ img.append(fpath(img[1][1:]))
+ else:
+ img.append(fpath(img[1]))
+
+ return img
+
+ for d in ['userguide', 'faq']:
+ for f in get_dir_rst_files(fpath(d)):
+ for category, slug, title, html_data, \
+ images, stderr in parse_rst_data(file(f).read(), d):
+ # absolute image paths
+ images = map(fix_image_path, images)
+ yield d, f, category, slug, title, html_data, images, stderr
+
+
+
--- /dev/null
+{% extends "admin/base_site.html" %}
+{% load i18n admin_modify adminmedia %}
+
+
+{% block extrahead %}{{ block.super }}
+{% url admin:jsi18n as jsi18nurl %}
+<script type="text/javascript" src="{{ jsi18nurl|default:"../../../jsi18n/" }}"></script>
+{{ media }}
+{% endblock %}
+
+{% block extrastyle %}
+{{ block.super }}
+<link rel="stylesheet" type="text/css" href="{% admin_media_prefix %}css/forms.css" />
+{% endblock %}
+
+{% block coltype %}{% endblock %}
+
+{% block bodyclass %}change-form{% endblock %}
+
+{% block breadcrumbs %}{% if not is_popup %}
+<div class="breadcrumbs">
+ <a href="../">{% trans "Home" %}</a> ›
+ <a href=".">{% trans "Import faq/guide from sphinx zip" %}</a> ›
+</div>
+{% endif %}{% endblock %}
+
+{% block content %}
+
+<div id="content-main">
+{% block object-tools %}
+{% if change %}{% if not is_popup %}
+ <ul class="object-tools"><li><a href="history/" class="historylink">{% trans "History" %}</a></li>
+ {% if has_absolute_url %}<li><a href="../../../r/{{ content_type_id }}/{{ object_id }}/" class="viewsitelink">{% trans "View on site" %}</a></li>{% endif%}
+ </ul>
+{% endif %}{% endif %}
+{% endblock %}
+<form enctype="multipart/form-data" action="." method="post" id="import_form">{% csrf_token %}{% block form_top %}{% endblock %}
+ <div>
+ {{ form.non_field_errors }}
+ {{ form.as_p }}
+ <input type="submit" />
+ </div>
+</form></div>
+{% endblock %}
urlpatterns = patterns('',
url(r'^feed/', BlogFeed(), name="blogfeed"),
+ url(r'^cmsmanage/sphinximport/$', 'cloudcms.admin.sphinx_import'),
url(r'^cmsmanage/', include(admin.site.urls)),
url(r'', include('feincms.urls')),
url(r'^sitemap\.xml$', 'django.contrib.sitemaps.views.sitemap', {'sitemaps': sitemaps }),
projects / snf-cloudcms / commitdiff