diff -r 0b389c93be20 -r 1e59c54b9652 TroubleTicket_db2db.py
--- a/TroubleTicket_db2db.py
+++ b/TroubleTicket_db2db.py
@@ -6,7 +6,7 @@
 # Author:      Panos Louridas, from an initial version by Spyros Kopsidas
 # Copyright:   (c) EGEE Project 2009
 # Licence:     Apache Software License
-
+#
 #-------------------------------------------------------------------------------
 
 import sys
@@ -20,11 +20,16 @@
 from TroubleTicketParser import *
 
 def get_unparsed_emails(cursor, table):
-    query = """SELECT Message_ID, Subject, Header, Body
-        FROM %s 
-        WHERE IS_STANDARDISED IS NULL"""
-#        "ORDER BY Date_Stored""" 
-    cursor.execute(query % (table))
+    """ A generator for returning unparsed emails."""
+    # Not using the more practical """ string because we want to
+    # be able to add easily the following line for testing:
+    # "AND header like '%grnet%' AND body like '%9966%' "
+    query = "SELECT Message_ID, Subject, Header, Body " + \
+        "FROM " + table + " " + \
+        "WHERE IS_STANDARDISED IS NULL " + \
+        "ORDER BY Date_Stored ASC"
+    cursor.execute(query)
+    print "Found %d raw mails." % cursor.rowcount
 
     while (1):
         row = cursor.fetchone()
@@ -39,6 +44,10 @@
         yield email
 
 def get_datetime_str(raw_datetime):
+    """A utility function for returning datetime objects out of strings.
+    Tries to parse them according to the following rules:
+    %d-%m-%Y
+    %Y-%m-%d"""
     try:
         dt = datetime.strptime(raw_datetime, "%d-%m-%Y")
     except ValueError:
@@ -48,11 +57,15 @@
             return None
     return dt.isoformat()
 
-def insert_values(values, db, destination_table, cursor, ttid):
+def insert_values(values, db, destination_table, connection, ttid):
+    """Inserts the normalised message represented by the values dictionary
+    to the database at the given destination_table. ttid is the original
+    message id in the unnormalised table."""
 
     # Find messages previously normalised with the same NREN / ID combination
     # as this will be an update. The previous ones will have their
     # MESSAGE_RNUMBER attribute increased by 1.
+    cursor = connection.cursor()
     original_id = int(values.get("ORIGINAL_ID", ""))
     partner_id = values.get("NREN", "")
     query = """SELECT ENOC_ID, MESSAGE_RNUMBER
@@ -60,12 +73,13 @@
         WHERE ORIGINAL_ID = %d AND PARTNER_ID = '%s'"""
     query = query % (destination_table, original_id, partner_id)
     cursor.execute(query)
-    while (1):
-        row = cursor.fetchone()
-        if row == None:
-            break
-        enoc_id = row[0]
-        message_rnumber = row[1]
+    # We need to fetchall to ensure that we have all the data in a
+    # snapshot in our program, so that the update that follows
+    # work correctly.
+    previous_messages = cursor.fetchall()
+    for message in previous_messages:
+        enoc_id = message[0]
+        message_rnumber = message[1]
         update_stmt = 'UPDATE ' + destination_table + ' ' + \
             'SET MESSAGE_RNUMBER = MESSAGE_RNUMBER + 1 ' + \
             'WHERE ENOC_ID  = ' + str(enoc_id)
@@ -73,6 +87,8 @@
         
     statement_pattern = "INSERT INTO " + destination_table + """
         (ORIGINAL_ID,
+        UN_ID,
+        PARTNER_ID,
         TT_OPEN_DATETIME, 
         OPEN_ENGINEER,
         CLOSE_ENGINEER, 
@@ -89,7 +105,7 @@
         ADDITIONAL_DATA, 
         HISTORY,
         MESSAGE_RNUMBER) 
-        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 
+        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 
         %s, %s, %s, %s, %s, %s, %s, %s)"""
           
     raw_datetime = values.get("TT_OPEN_DATETIME", "")
@@ -103,7 +119,8 @@
 
     statement_params = (
         original_id,
-        # partner_id,
+        ttid,
+        partner_id,
         tt_open_datetime,
         values.get("OPEN_ENGINEER", ""),
         values.get("CLOSE_ENGINEER", ""),
@@ -123,6 +140,8 @@
 
     # Insert the new normalised ticket.
     cursor.execute(statement_pattern, statement_params)
+    cursor.close()
+    connection.commit()
 
 def get_query_mark_as_parsed(row_id, db, source_table):
     return str("UPDATE `%s`.`%s`  SET `IS_STANDARDISED` = '1' WHERE `%s`.`ID` =%d LIMIT 1 ;" % (str(db), str(source_table), str(source_table), row_id))
@@ -158,7 +177,6 @@
 
     try:
         db2_conn = MySQLdb.connect(**TTConfig.TTdb_v2)
-        db2_cursor = db2_conn.cursor()
     except:
         print "Error connecting to database " + str(TTConfig.TTdb_v2['db'])
         exit()
@@ -180,22 +198,19 @@
         ttp.parse(raw_mail)
         parse_results = ttp.resultset
         num_found_keys = len(parse_results.keys())
-        if not 'ORIGINAL_ID' in parse_results:
-            continue
-        if not parse_results['ORIGINAL_ID'].isdigit():
-            continue
-        if num_found_keys >= TTConfig.configuration['min_keys_parsed']:
+        if ('ORIGINAL_ID' in parse_results
+            and parse_results['ORIGINAL_ID'].isdigit()
+            and num_found_keys >= TTConfig.configuration['min_keys_parsed']):
             insert_values(parse_results,
                           TTConfig.TTdb_v2['db'],
                           TTConfig.configuration['stdTickets_table'],
-                          db2_cursor,
+                          db2_conn,
                           raw_mail['ID'])
             rows_inserted = rows_inserted + 1
-        overwrite = overwrite_print("Mails read: %d; normalised"
+        overwrite = overwrite_print("Mails read: %d; normalised: %d"
                                     % (mails_read, rows_inserted),
                                     overwrite)
 
-
     db1_conn.close()            
     db2_conn.commit()
     db2_conn.close()
