code.grnet.gr Git - snf-cloudcms/blob - cloudcms/forms.py

   1 # -*- coding: utf-8 -*-
   2
   3 import zipfile
   4 import tempfile
   5 import os
   6 import glob
   7 import logging
   8 import StringIO
   9
  10 from django import forms
  11 from django.conf import settings
  12 from django.db import transaction
  13 from django.core.files import File
  14
  15 from feincms.module.medialibrary.models import Category as MediaCategory, \
  16         MediaFile
  17
  18 from cloudcmsguide.models import UserGuideEntry
  19 from cloudcmsfaq.models import Category as FaqCategory, Question
  20 from cloudcms.rstutils import generate_rst_contents_from_dir
  21 from cloudcms.models import Service, ServiceTranslation, Application
  22 from feincms.content.raw.models import RawContent
  23
  24 logger = logging.getLogger('cloudcms.rstimport')
  25
  26 def slugify(title):
  27     return title[:90]
  28
  29 # base filename to service slug map
  30 DEFAULT_SERVICE_MAP = {
  31         'cyclades':'cyclades',
  32         'okeanos':'okeanos',
  33         'pithos':'pithos'
  34 }
  35
  36 DEFAULT_REGION_MAP = {
  37         'faq':'main',
  38         'userguide':'main',
  39 }
  40
  41 DEFAULT_RESIZE_GEOMETRY = (400, 400)
  42
  43 SERVICE_MAP = getattr(settings, 'CMS_RST_IMPORT_SERVICE_FILE_MAP',
  44         DEFAULT_SERVICE_MAP)
  45 REGIONS_MAP = getattr(settings, 'CMS_RST_IMPORT_REGIONS_MAP',
  46         DEFAULT_REGION_MAP)
  47 RESIZE_GEOMETRY = getattr(settings, 'CMS_RST_IMPORT_RESIZE_GEOMETRY',
  48     DEFAULT_RESIZE_GEOMETRY)
  49
  50 def service_from_filename(rst):
  51     fname = os.path.basename(rst).replace(".rst","")
  52     service_slug = DEFAULT_SERVICE_MAP.get(fname, None)
  53     if not service_slug:
  54         return None
  55
  56     try:
  57         return Service.objects.filter(translations__slug=service_slug)[0]
  58     except IndexError:
  59         return None
  60     except Service.DoesNotExist:
  61         return None
  62
  63     return None
  64
  65
  66 def get_media_category(slug):
  67     return MediaCategory.objects.get(slug=slug)
  68
  69
  70 CATEGORIES_CHOICES = (('faq', 'FAQs'), ('user-guide', 'User guide'))
  71
  72 class RstZipImportForm(forms.Form):
  73
  74     FAQ_MEDIA_CATEGORY = 'faq-images'
  75     GUIDE_MEDIA_CATEGORY = 'user-guide-images'
  76
  77     clean_data = forms.BooleanField(initial=False, required=False)
  78     dry_run = forms.BooleanField(initial=True, required=False,
  79             widget=forms.HiddenInput)
  80     import_file = forms.FileField(required=True)
  81
  82     def __init__(self, *args, **kwargs):
  83         super(RstZipImportForm, self).__init__(*args, **kwargs)
  84         self.log_data = ""
  85
  86     def log(self, msg):
  87         self.log_data += "\n" + msg
  88
  89     def get_tmp_file(self, f):
  90         tmp = tempfile.mktemp('cloudcms-sphinx-import')
  91         f.file.reset()
  92         fd = file(tmp, 'w')
  93         fd.write(f.read())
  94         fd.close()
  95         return tmp
  96
  97     def clean_import_file(self):
  98         f = self.cleaned_data['import_file']
  99         tmpfile = self.get_tmp_file(f)
 100         if not zipfile.is_zipfile(tmpfile):
 101             raise forms.ValidationError("Invalid zip file")
 102         return f
 103
 104     def clean(self, *args, **kwargs):
 105         data = super(RstZipImportForm, self).clean(*args, **kwargs)
 106         return data
 107
 108     def clean_existing_data(self):
 109         logger.warning("Removing all FAQ questions")
 110         Question.objects.all().delete()
 111         logger.warning("Removing all User Guide entries")
 112         UserGuideEntry.objects.all().delete()
 113         logger.warning("Removing all media files in categories %s", [self.FAQ_MEDIA_CATEGORY,
 114             self.GUIDE_MEDIA_CATEGORY])
 115         MediaFile.objects.filter(categories__slug__in=[self.FAQ_MEDIA_CATEGORY, \
 116             self.GUIDE_MEDIA_CATEGORY]).delete()
 117
 118     def save(self, user, use_dir=None):
 119         stream = StringIO.StringIO()
 120         stream_handler = logging.StreamHandler(stream)
 121         stream_handler.setFormatter(logging.Formatter('<div class="log-entry %(levelname)s"><em>%(levelname)s</em>'
 122             '<pre>%(message)s</pre></div>'))
 123         logger.addHandler(stream_handler)
 124         old_level = logger.level
 125         logger.setLevel(logging.DEBUG)
 126
 127         dry_run = self.cleaned_data.get('dry_run')
 128         clean_data = self.cleaned_data.get('clean_data')
 129         import_file = self.cleaned_data.get('import_file')
 130
 131         if not use_dir:
 132             zipdir = tempfile.mkdtemp('cloudcms-sphinx-exports')
 133             zipdatafile = self.get_tmp_file(import_file)
 134             zipf = zipfile.ZipFile(file(zipdatafile))
 135             zipf.extractall(zipdir)
 136         else:
 137             zipdir = use_dir
 138
 139         subdirs = os.listdir(zipdir)
 140         if len(subdirs) == 1 and os.path.isdir(os.path.join(zipdir, \
 141                 subdirs[0])) and subdirs[0] != 'source':
 142             zipdir = os.path.join(zipdir, subdirs[0])
 143
 144         sid = transaction.savepoint()
 145
 146         if clean_data:
 147             try:
 148                 logger.warning("Removing exising entries")
 149                 self.clean_existing_data()
 150             except Exception, e:
 151                 try:
 152                     transaction.savepoint_rollback(sid)
 153                     transaction.rollback()
 154                 except Exception, e:
 155                     logger.exception("Can not rollback")
 156                 logger.exception("Failed to clean existing data")
 157                 logger.removeHandler(stream_handler)
 158                 logger.setLevel(old_level)
 159                 return False, stream.getvalue()
 160
 161         ret = ""
 162         try:
 163             logger.warning("Parsing contents of '%s'", zipdir)
 164             for data_type, rst, category, slug, title, html_content, \
 165                     images, stderr in generate_rst_contents_from_dir(zipdir):
 166
 167                 if stderr:
 168                     logger.error("Docutils error output for '%s'\n: %s" % (rst, stderr, ))
 169
 170                 service = service_from_filename(rst)
 171                 if not service:
 172                     logger.info("Skipping entry for file '%s'. No category found" % rst)
 173                     continue
 174
 175                 logger.info("Processing, '%s'" % (rst, ))
 176
 177
 178                 # first save media files
 179                 cat = False
 180                 newimages = []
 181                 if data_type == 'userguide':
 182                     cat = get_media_category(self.GUIDE_MEDIA_CATEGORY)
 183                 if data_type == 'faq':
 184                     cat = get_media_category(self.FAQ_MEDIA_CATEGORY)
 185
 186                 if not cat:
 187                     logger.info("Skipping %s, no media category found for '%s'",
 188                             rst, data_type)
 189                     continue
 190
 191                 for imgname, imgpath, imgabspath in images:
 192                     logger.debug("Checking image (%s, %s, %s)", imgname, imgpath, imgabspath)
 193                     newalt, newpath = create_or_update_media_file(cat, \
 194                             imgname, imgabspath)
 195
 196                     html_content = html_content.replace(imgpath, newpath)
 197
 198                 # now html contains correct image links, we are ready to save
 199                 # the faq/guide content
 200                 if data_type == 'faq':
 201                     logger.info('Processing FAQ entry, %s, %s, %s', service, slug, title)
 202                     cat = add_or_update_faq_category(unicode(category[0]),
 203                             unicode(category[1]))
 204                     question = add_or_update_faq_question(user, service, cat, slug, \
 205                             title, html_content)
 206
 207                 if data_type == 'userguide':
 208                     logger.info('Processing USER GUIDE entry, %s, %s, %s', service, slug, title)
 209                     guide_entry = add_or_update_guide_entry(user, service, slug, \
 210                             title, html_content)
 211
 212
 213         except Exception, e:
 214             print e, "EXCEPTION"
 215             logger.exception("RST import failed")
 216             logger.removeHandler(stream_handler)
 217             logger.setLevel(old_level)
 218             try:
 219                 transaction.savepoint_rollback(sid)
 220                 transaction.rollback()
 221             except Exception, e:
 222                 logger.exception("Can not rollback")
 223
 224             return False, stream.getvalue()
 225
 226         if dry_run:
 227             try:
 228                 transaction.savepoint_rollback(sid)
 229                 transaction.rollback()
 230             except Exception, e:
 231                 logger.exception("Can not rollback")
 232
 233         else:
 234             transaction.savepoint_commit(sid)
 235
 236         return True, stream.getvalue()
 237
 238
 239 def create_or_update_media_file(category, name, path):
 240     logger.info("Create or update media file, %s, %s, [category: %s]", name,
 241             path, category)
 242     name = category.title + " " + name
 243
 244     try:
 245         # TODO: Check language ?????
 246         mf = MediaFile.objects.get(categories__in=[category], translations__caption=name)
 247         logger.info("Media file found")
 248     except MediaFile.DoesNotExist:
 249         logger.info("Media file not found, creating...")
 250         mf = MediaFile()
 251         mf.file = File(open(path))
 252         mf.save()
 253         mf.translations.create(caption=name)
 254         mf.categories.clear()
 255         mf.categories = [category]
 256         mf.save()
 257
 258     # TODO: Check language ?????
 259     return mf.translations.all()[0].caption, mf.get_absolute_url()
 260
 261
 262 def add_or_update_faq_category(slug, name):
 263     logger.info("Create or update faq subcategory, %s, %s", slug,
 264             name)
 265     try:
 266         category = FaqCategory.objects.get(translations__slug=slug)
 267         logger.info("FAQ category found")
 268     except FaqCategory.DoesNotExist:
 269         logger.info("FAQ category not found, creating...")
 270         category = FaqCategory()
 271         category.save()
 272         category.translations.create(slug=slug, title=name)
 273
 274     return category
 275
 276
 277 def add_or_update_faq_question(author, service, category, slug,\
 278         title, html_content):
 279     logger.info("Create or update faq question, %s, %s, %s, %s", service,
 280             category, slug, title)
 281
 282     try:
 283         q = Question.objects.get(slug=slug)
 284         logger.info("Question found, updating...")
 285     except:
 286         logger.info("Question not found, creating...")
 287         q = Question()
 288
 289     q.author = author
 290     q.is_active = True
 291     q.category = category
 292     q.service = service
 293     q.slug = slugify(slug)
 294     q.title = unicode(title)
 295     q.save()
 296
 297     q.application = [Application.current()]
 298     q.save()
 299
 300     RawContentModel = Question.content_type_for(RawContent)
 301     try:
 302         content = q.rawcontent_set.filter()[0]
 303     except:
 304         content = q.rawcontent_set.create(region=REGIONS_MAP['faq'])
 305
 306     content.text = html_content
 307     content.save()
 308
 309     return q
 310
 311 def add_or_update_guide_entry(author, service, slug, title, html_content):
 312     logger.info("Create or update user guide entry, %s, %s, %s", service,
 313             slug, title)
 314     try:
 315         logger.info("Guide entry found, updating...")
 316         guide = UserGuideEntry.objects.get(slug=slug)
 317     except:
 318         logger.info("Guide entry not found, creating...")
 319         guide = UserGuideEntry()
 320
 321     guide.author = author
 322     guide.is_active = True
 323     guide.service = service
 324     guide.slug = slugify(slug)
 325     guide.title = unicode(title)
 326     guide.save()
 327
 328     RawContentModel = UserGuideEntry.content_type_for(RawContent)
 329     try:
 330         content = guide.rawcontent_set.filter()[0]
 331     except:
 332         content = guide.rawcontent_set.create(region=REGIONS_MAP['userguide'])
 333
 334     content.text = html_content
 335     content.save()
 336
 337     return guide
 338
 339