commit 58b9c19600ac95fd9d7b1a6c4fc13ea5fb1c390e
Author: Kostas Papadimitriou <kpap@grnet.gr>
Date:   Mon Jul 16 18:18:55 2012 +0300

    Sphinx docs import form improvements
    
    - Custom stream logging handler to capture logging output of the
      convertion process.
    - Fix media object creation/update (proper categories handling)

diff --git a/cloudcms/forms.py b/cloudcms/forms.py
index 0d127fd..2737fcb 100644
--- a/cloudcms/forms.py
+++ b/cloudcms/forms.py
@@ -5,6 +5,7 @@ import tempfile
 import os
 import glob
 import logging
+import StringIO
 
 from django import forms
 from django.conf import settings
@@ -63,7 +64,7 @@ def get_media_category(slug):
     return MediaCategory.objects.get(slug=slug)
 
 
-CATEGORIES_CHOICES = (('faqs', 'FAQs'), ('guide', 'User guide'))
+CATEGORIES_CHOICES = (('faq', 'FAQs'), ('user-guide', 'User guide'))
 
 class RstZipImportForm(forms.Form):
 
@@ -102,12 +103,24 @@ class RstZipImportForm(forms.Form):
         return data
 
     def clean_existing_data(self):
+        logger.warning("Removing all FAQ questions")
         Question.objects.all().delete()
+        logger.warning("Removing all User Guide entries")
         UserGuideEntry.objects.all().delete()
+        logger.warning("Removing all media files in categories %s", [self.FAQ_MEDIA_CATEGORY,
+            self.GUIDE_MEDIA_CATEGORY])
         MediaFile.objects.filter(categories__slug__in=[self.FAQ_MEDIA_CATEGORY, \
             self.GUIDE_MEDIA_CATEGORY]).delete()
 
     def save(self, user, use_dir=None):
+        stream = StringIO.StringIO()
+        stream_handler = logging.StreamHandler(stream)
+        stream_handler.setFormatter(logging.Formatter('<div class="log-entry %(levelname)s"><em>%(levelname)s</em>'
+            '<pre>%(message)s</pre></div>'))
+        logger.addHandler(stream_handler)
+        old_level = logger.level
+        logger.setLevel(logging.DEBUG)
+
         dry_run = self.cleaned_data.get('dry_run')
         clean_data = self.cleaned_data.get('clean_data')
         import_file = self.cleaned_data.get('import_file')
@@ -125,32 +138,38 @@ class RstZipImportForm(forms.Form):
                 subdirs[0])) and subdirs[0] != 'source':
             zipdir = os.path.join(zipdir, subdirs[0])
 
-        #sid = transaction.savepoint()
+        sid = transaction.savepoint()
 
         if clean_data:
             try:
+                logger.warning("Removing exising entries")
                 self.clean_existing_data()
             except Exception, e:
-                return False
+                transaction.savepoint_rollback(sid)
+                logger.exception("Failed to clean existing data")
+                logger.removeHandler(stream_handler)
+                logger.setLevel(old_level)
+                return False, stream.getvalue()
 
         ret = ""
         try:
+            logger.warning("Parsing contents of '%s'", zipdir)
             for data_type, rst, category, slug, title, html_content, \
                     images, stderr in generate_rst_contents_from_dir(zipdir):
 
-                ret += stderr
-                #logger.info("Parsed %s" % (rst, ))
                 if stderr:
-                    pass
-                    #logger.info("Error output: %s" % (stderr, ))
+                    logger.error("Docutils error output for '%s'\n: %s" % (rst, stderr, ))
 
                 service = service_from_filename(rst)
                 if not service:
                     logger.info("Skipping entry for file '%s'. No category found" % rst)
                     continue
 
+                logger.info("Processing, '%s'" % (rst, ))
+
 
                 # first save media files
+                cat = False
                 newimages = []
                 if data_type == 'userguide':
                     cat = get_media_category(self.GUIDE_MEDIA_CATEGORY)
@@ -158,9 +177,12 @@ class RstZipImportForm(forms.Form):
                     cat = get_media_category(self.FAQ_MEDIA_CATEGORY)
 
                 if not cat:
+                    logger.info("Skipping %s, no media category found for '%s'",
+                            rst, data_type)
                     continue
 
                 for imgname, imgpath, imgabspath in images:
+                    logger.debug("Checking image (%s, %s, %s)", imgname, imgpath, imgabspath)
                     newalt, newpath = create_or_update_media_file(cat, \
                             imgname, imgabspath)
 
@@ -169,61 +191,80 @@ class RstZipImportForm(forms.Form):
                 # now html contains correct image links, we are ready to save
                 # the faq/guide content
                 if data_type == 'faq':
+                    logger.info('Processing FAQ entry, %s, %s, %s', service, slug, title)
                     cat = add_or_update_faq_category(category[0], category[1])
                     question = add_or_update_faq_question(user, service, cat, slug, \
                             title, html_content)
 
                 if data_type == 'userguide':
+                    logger.info('Processing USER GUIDE entry, %s, %s, %s', service, slug, title)
                     guide_entry = add_or_update_guide_entry(user, service, slug, \
                             title, html_content)
 
 
         except Exception, e:
             logger.exception("RST import failed")
-            #transaction.savepoint_rollback(sid)
-            return False
+            logger.removeHandler(stream_handler)
+            logger.setLevel(old_level)
+            transaction.savepoint_rollback(sid)
+            return False, stream.getvalue()
 
         if dry_run:
-            pass
-            #transaction.savepoint_rollback(sid)
+            transaction.savepoint_rollback(sid)
         else:
-            pass
-            #transaction.savepoint_commit(sid)
+            transaction.savepoint_commit(sid)
+
+        return True, stream.getvalue()
 
 
 def create_or_update_media_file(category, name, path):
+    logger.info("Create or update media file, %s, %s, [category: %s]", name,
+            path, category)
     name = category.title + " " + name
 
     try:
         # TODO: Check language ?????
-        mf = MediaFile.objects.get(categories=category, translations__caption=name)
+        mf = MediaFile.objects.get(categories__in=[category], translations__caption=name)
+        logger.info("Media file found")
     except MediaFile.DoesNotExist:
+        logger.info("Media file not found, creating...")
         mf = MediaFile()
-        mf.category = category
         mf.file = File(open(path))
         mf.save()
         mf.translations.create(caption=name)
+        mf.categories.clear()
+        mf.categories = [category]
+        mf.save()
 
     # TODO: Check language ?????
     return mf.translations.all()[0].caption, mf.get_absolute_url()
 
 
 def add_or_update_faq_category(slug, name):
+    logger.info("Create or update faq subcategory, %s, %s", slug,
+            name)
     try:
         category = FaqCategory.objects.get(translations__slug=slug)
+        logger.info("FAQ category found")
     except FaqCategory.DoesNotExist:
+        logger.info("FAQ category not found, creating...")
         category = FaqCategory()
         category.save()
         category.translations.create(slug=slug, title=name)
 
     return category
 
+
 def add_or_update_faq_question(author, service, category, slug,\
         title, html_content):
+    logger.info("Create or update faq question, %s, %s, %s, %s", service,
+            category, slug, title)
 
     try:
         q = Question.objects.get(slug=slug)
+        logger.info("Question found, updating...")
     except:
+        logger.info("Question not found, creating...")
         q = Question()
 
     q.author = author
@@ -248,9 +289,13 @@ def add_or_update_faq_question(author, service, category, slug,\
     return q
 
 def add_or_update_guide_entry(author, service, slug, title, html_content):
+    logger.info("Create or update user guide entry, %s, %s, %s", service,
+            slug, title)
     try:
+        logger.info("Guide entry found, updating...")
         guide = UserGuideEntry.objects.get(slug=slug)
     except:
+        logger.info("Guide entry not found, creating...")
         guide = UserGuideEntry()
 
     guide.author = author
@@ -271,3 +316,4 @@ def add_or_update_guide_entry(author, service, slug, title, html_content):
 
     return guide
 
+
