code.grnet.gr Git - snf-cloudcms/blob - cloudcms/forms.py

   1 # -*- coding: utf-8 -*-
   2
   3 import zipfile
   4 import tempfile
   5 import os
   6 import glob
   7 import logging
   8 import StringIO
   9
  10 from django import forms
  11 from django.conf import settings
  12 from django.db import transaction
  13 from django.core.files import File
  14
  15 from feincms.module.medialibrary.models import Category as MediaCategory, \
  16         MediaFile
  17
  18 from cloudcmsguide.models import UserGuideEntry
  19 from cloudcmsfaq.models import Category as FaqCategory, Question
  20 from cloudcms.rstutils import generate_rst_contents_from_dir
  21 from cloudcms.models import Service, ServiceTranslation, Application
  22 from feincms.content.raw.models import RawContent
  23
  24 logger = logging.getLogger('cloudcms.rstimport')
  25
  26 def slugify(title):
  27     return title[:90]
  28
  29 # base filename to service slug map
  30 DEFAULT_SERVICE_MAP = {
  31         'cyclades':'cyclades',
  32         'okeanos':'okeanos',
  33         'pithos':'pithos'
  34 }
  35
  36 DEFAULT_REGION_MAP = {
  37         'faq':'main',
  38         'userguide':'main',
  39 }
  40
  41 DEFAULT_RESIZE_GEOMETRY = (400, 400)
  42
  43 SERVICE_MAP = getattr(settings, 'CMS_RST_IMPORT_SERVICE_FILE_MAP',
  44         DEFAULT_SERVICE_MAP)
  45 REGIONS_MAP = getattr(settings, 'CMS_RST_IMPORT_REGIONS_MAP',
  46         DEFAULT_REGION_MAP)
  47 RESIZE_GEOMETRY = getattr(settings, 'CMS_RST_IMPORT_RESIZE_GEOMETRY',
  48     DEFAULT_RESIZE_GEOMETRY)
  49
  50 def service_from_filename(rst):
  51     fname = os.path.basename(rst).replace(".rst","")
  52     service_slug = SERVICE_MAP.get(fname, None)
  53     if not service_slug:
  54         return None
  55
  56     try:
  57         return Service.objects.filter(translations__slug=service_slug)[0]
  58     except IndexError:
  59         return None
  60     except Service.DoesNotExist:
  61         return None
  62
  63     return None
  64
  65
  66 def get_media_category(slug):
  67     return MediaCategory.objects.get(slug=slug)
  68
  69
  70 CATEGORIES_CHOICES = (('faq', 'FAQs'), ('user-guide', 'User guide'))
  71
  72 class RstZipImportForm(forms.Form):
  73
  74     FAQ_MEDIA_CATEGORY = 'faq-images'
  75     GUIDE_MEDIA_CATEGORY = 'user-guide-images'
  76
  77     clean_data = forms.BooleanField(initial=False, required=False)
  78     dry_run = forms.BooleanField(initial=True, required=False,
  79             widget=forms.HiddenInput)
  80     import_file = forms.FileField(required=True)
  81
  82     def __init__(self, *args, **kwargs):
  83         super(RstZipImportForm, self).__init__(*args, **kwargs)
  84         self.log_data = ""
  85
  86     def log(self, msg):
  87         self.log_data += "\n" + msg
  88
  89     def get_tmp_file(self, f):
  90         tmp = tempfile.mktemp('cloudcms-sphinx-import')
  91         f.file.reset()
  92         fd = file(tmp, 'w')
  93         fd.write(f.read())
  94         fd.close()
  95         return tmp
  96
  97     def clean_import_file(self):
  98         f = self.cleaned_data['import_file']
  99         tmpfile = self.get_tmp_file(f)
 100         if not zipfile.is_zipfile(tmpfile):
 101             raise forms.ValidationError("Invalid zip file")
 102         return f
 103
 104     def clean(self, *args, **kwargs):
 105         data = super(RstZipImportForm, self).clean(*args, **kwargs)
 106         return data
 107
 108     def clean_existing_data(self):
 109         logger.warning("Removing all FAQ questions")
 110         Question.objects.all().delete()
 111         logger.warning("Removing all User Guide entries")
 112         UserGuideEntry.objects.all().delete()
 113         logger.warning("Removing all media files in categories %s", [self.FAQ_MEDIA_CATEGORY,
 114             self.GUIDE_MEDIA_CATEGORY])
 115         MediaFile.objects.filter(categories__slug__in=[self.FAQ_MEDIA_CATEGORY, \
 116             self.GUIDE_MEDIA_CATEGORY]).delete()
 117
 118     def save(self, user, use_dir=None):
 119         stream = StringIO.StringIO()
 120         stream_handler = logging.StreamHandler(stream)
 121         stream_handler.setFormatter(logging.Formatter('<div class="log-entry %(levelname)s"><em>%(levelname)s</em>'
 122             '<pre>%(message)s</pre></div>'))
 123         logger.addHandler(stream_handler)
 124         old_level = logger.level
 125         logger.setLevel(logging.DEBUG)
 126
 127         dry_run = self.cleaned_data.get('dry_run')
 128         clean_data = self.cleaned_data.get('clean_data')
 129         import_file = self.cleaned_data.get('import_file')
 130
 131         if not use_dir:
 132             zipdir = tempfile.mkdtemp('cloudcms-sphinx-exports')
 133             zipdatafile = self.get_tmp_file(import_file)
 134             zipf = zipfile.ZipFile(file(zipdatafile))
 135             zipf.extractall(zipdir)
 136         else:
 137             zipdir = use_dir
 138
 139         subdirs = os.listdir(zipdir)
 140         if len(subdirs) == 1 and os.path.isdir(os.path.join(zipdir, \
 141                 subdirs[0])) and subdirs[0] != 'source':
 142             zipdir = os.path.join(zipdir, subdirs[0])
 143
 144         sid = transaction.savepoint()
 145
 146         if clean_data:
 147             try:
 148                 logger.warning("Removing exising entries")
 149                 self.clean_existing_data()
 150             except Exception, e:
 151                 try:
 152                     transaction.savepoint_rollback(sid)
 153                 except Exception, e:
 154                     logger.exception("Can not rollback")
 155                 logger.exception("Failed to clean existing data")
 156                 logger.removeHandler(stream_handler)
 157                 logger.setLevel(old_level)
 158                 return False, stream.getvalue()
 159
 160         ret = ""
 161         try:
 162             logger.warning("Parsing contents of '%s'", zipdir)
 163             for data_type, rst, category, slug, title, html_content, \
 164                     images, stderr in generate_rst_contents_from_dir(zipdir):
 165
 166                 if stderr:
 167                     logger.error("Docutils error output for '%s'\n: %s" % (rst, stderr, ))
 168
 169                 service = service_from_filename(rst)
 170                 if not service:
 171                     logger.info("Skipping entry for file '%s'. No category found" % rst)
 172                     continue
 173
 174                 logger.info("Processing, '%s'" % (rst, ))
 175
 176
 177                 # first save media files
 178                 cat = False
 179                 newimages = []
 180                 if data_type == 'userguide':
 181                     cat = get_media_category(self.GUIDE_MEDIA_CATEGORY)
 182                 if data_type == 'faq':
 183                     cat = get_media_category(self.FAQ_MEDIA_CATEGORY)
 184
 185                 if not cat:
 186                     logger.info("Skipping %s, no media category found for '%s'",
 187                             rst, data_type)
 188                     continue
 189
 190                 for imgname, imgpath, imgabspath in images:
 191                     logger.debug("Checking image (%s, %s, %s)", imgname, imgpath, imgabspath)
 192                     newalt, newpath = create_or_update_media_file(cat, \
 193                             imgname, imgabspath)
 194
 195                     html_content = html_content.replace(imgpath, newpath)
 196
 197                 # now html contains correct image links, we are ready to save
 198                 # the faq/guide content
 199                 if data_type == 'faq':
 200                     logger.info('Processing FAQ entry, %s, %s, %s', service, slug, title)
 201                     cat = add_or_update_faq_category(unicode(category[0]),
 202                             unicode(category[1]))
 203                     question = add_or_update_faq_question(user, service, cat, slug, \
 204                             title, html_content)
 205
 206                 if data_type == 'userguide':
 207                     logger.info('Processing USER GUIDE entry, %s, %s, %s', service, slug, title)
 208                     guide_entry = add_or_update_guide_entry(user, service, slug, \
 209                             title, html_content)
 210
 211
 212         except Exception, e:
 213             print e, "EXCEPTION"
 214             logger.exception("RST import failed")
 215             logger.removeHandler(stream_handler)
 216             logger.setLevel(old_level)
 217             try:
 218                 transaction.savepoint_rollback(sid)
 219                 transaction.rollback()
 220             except Exception, e:
 221                 logger.exception("Can not rollback")
 222
 223             return False, stream.getvalue()
 224
 225         if dry_run:
 226             try:
 227                 transaction.savepoint_rollback(sid)
 228                 transaction.rollback()
 229             except Exception, e:
 230                 logger.exception("Can not rollback")
 231
 232         else:
 233             transaction.savepoint_commit(sid)
 234
 235         return True, stream.getvalue()
 236
 237
 238 def create_or_update_media_file(category, name, path):
 239     logger.info("Create or update media file, %s, %s, [category: %s]", name,
 240             path, category)
 241     name = category.title + " " + name
 242
 243     try:
 244         # TODO: Check language ?????
 245         mf = MediaFile.objects.get(categories__in=[category], translations__caption=name)
 246         logger.info("Media file found")
 247     except MediaFile.DoesNotExist:
 248         logger.info("Media file not found, creating...")
 249         mf = MediaFile()
 250         mf.file = File(open(path))
 251         mf.save()
 252         mf.translations.create(caption=name)
 253         mf.categories.clear()
 254         mf.categories = [category]
 255         mf.save()
 256
 257     # TODO: Check language ?????
 258     return mf.translations.all()[0].caption, mf.get_absolute_url()
 259
 260
 261 def add_or_update_faq_category(slug, name):
 262     logger.info("Create or update faq subcategory, %s, %s", slug,
 263             name)
 264     try:
 265         category = FaqCategory.objects.get(translations__slug=slug)
 266         logger.info("FAQ category found")
 267     except FaqCategory.DoesNotExist:
 268         logger.info("FAQ category not found, creating...")
 269         category = FaqCategory()
 270         category.save()
 271         category.translations.create(slug=slug, title=name)
 272
 273     return category
 274
 275
 276 def add_or_update_faq_question(author, service, category, slug,\
 277         title, html_content):
 278     logger.info("Create or update faq question, %s, %s, %s, %s", service,
 279             category, slug, title)
 280
 281     try:
 282         q = Question.objects.get(slug=slug)
 283         logger.info("Question found, updating...")
 284     except:
 285         logger.info("Question not found, creating...")
 286         q = Question()
 287
 288     q.author = author
 289     q.is_active = True
 290     q.category = category
 291     q.service = service
 292     q.slug = slugify(slug)
 293     q.title = unicode(title)
 294     q.save()
 295
 296     q.application = [Application.current()]
 297     q.save()
 298
 299     RawContentModel = Question.content_type_for(RawContent)
 300     try:
 301         content = q.rawcontent_set.filter()[0]
 302     except:
 303         content = q.rawcontent_set.create(region=REGIONS_MAP['faq'])
 304
 305     content.text = html_content
 306     content.save()
 307
 308     return q
 309
 310 def add_or_update_guide_entry(author, service, slug, title, html_content):
 311     logger.info("Create or update user guide entry, %s, %s, %s", service,
 312             slug, title)
 313     try:
 314         logger.info("Guide entry found, updating...")
 315         guide = UserGuideEntry.objects.get(slug=slug)
 316     except:
 317         logger.info("Guide entry not found, creating...")
 318         guide = UserGuideEntry()
 319
 320     guide.author = author
 321     guide.is_active = True
 322     guide.service = service
 323     guide.slug = slugify(slug)
 324     guide.title = unicode(title)
 325     guide.save()
 326
 327     RawContentModel = UserGuideEntry.content_type_for(RawContent)
 328     try:
 329         content = guide.rawcontent_set.filter()[0]
 330     except:
 331         content = guide.rawcontent_set.create(region=REGIONS_MAP['userguide'])
 332
 333     content.text = html_content
 334     content.save()
 335
 336     return guide
 337
 338