Synnefo » snf-cloudcms

# -*- coding: utf-8 -*-

import zipfile

import tempfile

import os

import glob

import logging

import StringIO

from django import forms

from django.conf import settings

from django.db import transaction

from django.core.files import File

from feincms.module.medialibrary.models import Category as MediaCategory, \

        MediaFile

from cloudcmsguide.models import UserGuideEntry

from cloudcmsfaq.models import Category as FaqCategory, Question

from cloudcms.rstutils import generate_rst_contents_from_dir

from cloudcms.models import Service, ServiceTranslation, Application

from feincms.content.raw.models import RawContent

logger = logging.getLogger('cloudcms.rstimport')

def slugify(title):

    return title[:90]

# base filename to service slug map

DEFAULT_SERVICE_MAP = {

        'cyclades':'cyclades',

        'okeanos':'okeanos',

        'pithos':'pithos'

}

DEFAULT_REGION_MAP = {

        'faq':'main',

        'userguide':'main',

}

DEFAULT_RESIZE_GEOMETRY = (400, 400)

SERVICE_MAP = getattr(settings, 'CMS_RST_IMPORT_SERVICE_FILE_MAP',

        DEFAULT_SERVICE_MAP)

REGIONS_MAP = getattr(settings, 'CMS_RST_IMPORT_REGIONS_MAP',

        DEFAULT_REGION_MAP)

RESIZE_GEOMETRY = getattr(settings, 'CMS_RST_IMPORT_RESIZE_GEOMETRY',

    DEFAULT_RESIZE_GEOMETRY)

def service_from_filename(rst):

    fname = os.path.basename(rst).replace(".rst","")

    service_slug = SERVICE_MAP.get(fname, None)

    if not service_slug:

        return None

    try:

        return Service.objects.filter(translations__slug=service_slug)[0]

    except IndexError:

        return None

    except Service.DoesNotExist:

        return None

    return None

def get_media_category(slug):

    return MediaCategory.objects.get(slug=slug)

CATEGORIES_CHOICES = (('faq', 'FAQs'), ('user-guide', 'User guide'))

class RstZipImportForm(forms.Form):

    FAQ_MEDIA_CATEGORY = 'faq-images'

    GUIDE_MEDIA_CATEGORY = 'user-guide-images'

    clean_data = forms.BooleanField(initial=False, required=False)

    dry_run = forms.BooleanField(initial=True, required=False,

            widget=forms.HiddenInput)

    import_file = forms.FileField(required=True)

    def __init__(self, *args, **kwargs):

        super(RstZipImportForm, self).__init__(*args, **kwargs)

        self.log_data = ""

    def log(self, msg):

        self.log_data += "\n" + msg

    def get_tmp_file(self, f):

        tmp = tempfile.mktemp('cloudcms-sphinx-import')

        f.seek(0)

        fd = file(tmp, 'w')

        fd.write(f.read())

        fd.close()

        return tmp

    def clean_import_file(self):

        f = self.cleaned_data['import_file']

        tmpfile = self.get_tmp_file(f)

        if not zipfile.is_zipfile(tmpfile):

            raise forms.ValidationError("Invalid zip file")

        return f

    def clean(self, *args, **kwargs):

        data = super(RstZipImportForm, self).clean(*args, **kwargs)

        return data

    def clean_existing_data(self):

        logger.warning("Removing all FAQ questions")

        Question.objects.all().delete()

        logger.warning("Removing all User Guide entries")

        UserGuideEntry.objects.all().delete()

        logger.warning("Removing all media files in categories %s", [self.FAQ_MEDIA_CATEGORY,

            self.GUIDE_MEDIA_CATEGORY])

        MediaFile.objects.filter(categories__slug__in=[self.FAQ_MEDIA_CATEGORY, \

            self.GUIDE_MEDIA_CATEGORY]).delete()

    def save(self, user, use_dir=None):

        stream = StringIO.StringIO()

        stream_handler = logging.StreamHandler(stream)

        stream_handler.setFormatter(logging.Formatter('<div class="log-entry %(levelname)s"><em>%(levelname)s</em>'

            '<pre>%(message)s</pre></div>'))

        logger.addHandler(stream_handler)

        old_level = logger.level

        logger.setLevel(logging.DEBUG)

        dry_run = self.cleaned_data.get('dry_run')

        clean_data = self.cleaned_data.get('clean_data')

        import_file = self.cleaned_data.get('import_file')

        if not use_dir:

            zipdir = tempfile.mkdtemp('cloudcms-sphinx-exports')

            zipdatafile = self.get_tmp_file(import_file)

            zipf = zipfile.ZipFile(file(zipdatafile))

            zipf.extractall(zipdir)

        else:

            zipdir = use_dir

        subdirs = os.listdir(zipdir)

        if len(subdirs) == 1 and os.path.isdir(os.path.join(zipdir, \

                subdirs[0])) and subdirs[0] != 'source':

            zipdir = os.path.join(zipdir, subdirs[0])

        sid = transaction.savepoint()

        if clean_data:

            try:

                logger.warning("Removing exising entries")

                self.clean_existing_data()

            except Exception, e:

                try:

                    transaction.savepoint_rollback(sid)

                except Exception, e:

                    logger.exception("Can not rollback")

                logger.exception("Failed to clean existing data")

                logger.removeHandler(stream_handler)

                logger.setLevel(old_level)

                return False, stream.getvalue()

        ret = ""

        try:

            logger.warning("Parsing contents of '%s'", zipdir)

            for data_type, rst, category, slug, title, html_content, \

                    images, stderr in generate_rst_contents_from_dir(zipdir):

                if stderr:

                    logger.error("Docutils error output for '%s'\n: %s" % (rst, stderr, ))

                service = service_from_filename(rst)

                if not service:

                    logger.info("Skipping entry for file '%s'. No category found" % rst)

                    continue

                logger.info("Processing, '%s'" % (rst, ))

                # first save media files

                cat = False

                newimages = []

                if data_type == 'userguide':

                    cat = get_media_category(self.GUIDE_MEDIA_CATEGORY)

                if data_type == 'faq':

                    cat = get_media_category(self.FAQ_MEDIA_CATEGORY)

                if not cat:

                    logger.info("Skipping %s, no media category found for '%s'",

                            rst, data_type)

                    continue

                for imgname, imgpath, imgabspath in images:

                    logger.debug("Checking image (%s, %s, %s)", imgname, imgpath, imgabspath)

                    newalt, newpath = create_or_update_media_file(cat, \

                            imgname, imgabspath)

                    html_content = html_content.replace(imgpath, newpath)

                # now html contains correct image links, we are ready to save

                # the faq/guide content

                if data_type == 'faq':

                    logger.info('Processing FAQ entry, %s, %s, %s', service, slug, title)

                    cat = add_or_update_faq_category(unicode(category[0]),

                            unicode(category[1]))

                    question = add_or_update_faq_question(user, service, cat, slug, \

                            title, html_content)

                if data_type == 'userguide':

                    logger.info('Processing USER GUIDE entry, %s, %s, %s', service, slug, title)

                    guide_entry = add_or_update_guide_entry(user, service, slug, \

                            title, html_content)

        except Exception, e:

            print e, "EXCEPTION"

            logger.exception("RST import failed")

            logger.removeHandler(stream_handler)

            logger.setLevel(old_level)

            try:

                transaction.savepoint_rollback(sid)

                transaction.rollback()

            except Exception, e:

                logger.exception("Can not rollback")

            return False, stream.getvalue()

        if dry_run:

            try:

                transaction.savepoint_rollback(sid)

                transaction.rollback()

            except Exception, e:

                logger.exception("Can not rollback")

        else:

            transaction.savepoint_commit(sid)

        return True, stream.getvalue()

def create_or_update_media_file(category, name, path):

    logger.info("Create or update media file, %s, %s, [category: %s]", name,

            path, category)

    name = category.title + " " + name

    try:

        # TODO: Check language ?????

        mf = MediaFile.objects.get(categories__in=[category], translations__caption=name)

        logger.info("Media file found")

    except MediaFile.DoesNotExist:

        logger.info("Media file not found, creating...")

        mf = MediaFile()

        mf.file = File(open(path))

        mf.save()

        mf.translations.create(caption=name)

        mf.categories.clear()

        mf.categories = [category]

        mf.save()

    # TODO: Check language ?????

    return mf.translations.all()[0].caption, mf.get_absolute_url()

def add_or_update_faq_category(slug, name):

    logger.info("Create or update faq subcategory, %s, %s", slug,

            name)

    try:

        category = FaqCategory.objects.get(translations__slug=slug)

        logger.info("FAQ category found")

    except FaqCategory.DoesNotExist:

        logger.info("FAQ category not found, creating...")

        category = FaqCategory()

        category.save()

        category.translations.create(slug=slug, title=name)

    return category

def add_or_update_faq_question(author, service, category, slug,\

        title, html_content):

    logger.info("Create or update faq question, %s, %s, %s, %s", service,

            category, slug, title)

    try:

        q = Question.objects.get(slug=slug)

        logger.info("Question found, updating...")

    except:

        logger.info("Question not found, creating...")

        q = Question()

    q.author = author

    q.is_active = True

    q.category = category

    q.service = service

    q.slug = slugify(slug)

    q.title = unicode(title)

    q.save()

    q.application = [Application.current()]

    q.save()

    RawContentModel = Question.content_type_for(RawContent)

    try:

        content = q.rawcontent_set.filter()[0]

    except:

        content = q.rawcontent_set.create(region=REGIONS_MAP['faq'])

    content.text = html_content

    content.save()

    return q

def add_or_update_guide_entry(author, service, slug, title, html_content):

    logger.info("Create or update user guide entry, %s, %s, %s", service,

            slug, title)

    try:

        logger.info("Guide entry found, updating...")

        guide = UserGuideEntry.objects.get(slug=slug)

    except:

        logger.info("Guide entry not found, creating...")

        guide = UserGuideEntry()

    guide.author = author

    guide.is_active = True

    guide.service = service

    guide.slug = slugify(slug)

    guide.title = unicode(title)

    guide.save()

    RawContentModel = UserGuideEntry.content_type_for(RawContent)

    try:

        content = guide.rawcontent_set.filter()[0]

    except:

        content = guide.rawcontent_set.create(region=REGIONS_MAP['userguide'])

    content.text = html_content

    content.save()

    return guide