Ticket #1462: bugzilla2trac.py.4.diff

bugzilla2trac.py

@@ -13 +13 @@
 Copyright 2004, Dmitry Yusupov <dmitry_yus@yahoo.com>
 
 Many enhancements, Bill Soudan <bill@soudan.net>
+Other enhancements, Florent Guillaume <fg@nuxeo.com>
 """
 
+import re
+
 ###
 ### Conversion Settings -- edit these before running if desired
 ###
@@ -22 +25 @@
 # Bugzilla version.  You can find this in Bugzilla's globals.pl file.
 #
 # Currently, the following bugzilla versions are known to work:
-#   2.11
+#   2.11, 2.16.5
 #
 # If you run this script on a version not listed here and it is successful,
 # please report it to the Trac mailing list so we can update the list.
@@ -31 +34 @@
 # MySQL connection parameters for the Bugzilla database.  These can also 
 # be specified on the command line.
 BZ_DB = ''
-BZ_HOST = 'localhost'
+BZ_HOST = ''
 BZ_USER = ''
 BZ_PASSWORD = ''
 
@@ -40 +43 @@
 
 # If true, all existing Trac tickets and attachments will be removed 
 # prior to import.
-TRAC_CLEAN = False
+TRAC_CLEAN = FALSE
 
 # Enclose imported ticket description and comments in a {{{ }}} 
 # preformat block?  This formats the text in a fixed-point font.
 PREFORMAT_COMMENTS = False
 
+# Replace bug numbers in comments with #xyz
+REPLACE_BUG_NO = False
+
+# Severities
+SEVERITIES = [
+    ('blocker',     '1'),
+    ('critical',    '2'),
+    ('major',       '3'),
+    ('normal',      '4'),
+    ('minor',       '5'),
+    ('trivial',     '6'),
+    ('enhancement', '7'),
+    ]
+
+# Priorities
+PRIORITIES = [
+    ('P1', '1'),
+    ('P2', '2'),
+    ('P3', '3'),
+    ('P4', '4'),
+    ('P5', '5'),
+    ]
+
 # By default, all bugs are imported from Bugzilla.  If you add a list
 # of products here, only bugs from those products will be imported.
 PRODUCTS = []
+# These Bugzilla products will be ignored during import.
+IGNORE_PRODUCTS = []
 
-# Trac doesn't have the concept of a product.  Instead, this script can
-# assign keywords in the ticket entry to represent products.
-#
-# ex. PRODUCT_KEYWORDS = { 'product1' : 'PRODUCT1_KEYWORD' }
-PRODUCT_KEYWORDS = {}
+# These milestones are ignored
+IGNORE_MILESTONES = ['---']
 
+# These logins are converted to these user ids
+LOGIN_MAP = {
+    #'some.user@example.com': 'someuser',
+   }
+
+# These emails are removed from CC list
+IGNORE_CC = [
+    #'loser@example.com',
+    ]
+
+# The 'component' field in Trac can come either from the Product or
+# or from the Component field of Bugzilla. COMPONENTS_FROM_PRODUCTS
+# switches the behavior.
+# If COMPONENTS_FROM_PRODUCTS is True:
+# - Bugzilla Product -> Trac Component
+# - Bugzilla Component -> Trac Keyword
+# IF COMPONENTS_FROM_PRODUCTS is False:
+# - Bugzilla Product -> Trac Keyword
+# - Bugzilla Component -> Trac Component
+COMPONENTS_FROM_PRODUCTS = False
+
+# If COMPONENTS_FROM_PRODUCTS is True, the default owner for each
+# Trac component is inferred from a default Bugzilla component.
+DEFAULT_COMPONENTS = ['default', 'misc', 'main']
+
+# This mapping can assign keywords in the ticket entry to represent
+# products or components (depending on COMPONENTS_FROM_PRODUCTS).
+# The keyword will be ignored if empty.
+KEYWORDS_MAPPING = {
+    #'Bugzilla_product_or_component': 'Keyword',
+    'default': '',
+    'misc': '',
+    }
+
+# If this is True, products or components are all set as keywords
+# even if not mentionned in KEYWORDS_MAPPING.
+MAP_ALL_KEYWORDS = True
+
+
 # Bug comments that should not be imported.  Each entry in list should
 # be a regular expression.
 IGNORE_COMMENTS = [
@@ -96 +160 @@
 # otherwise you'd see changes for fields that don't exist in Trac.
 IGNORED_ACTIVITY_FIELDS = ['everconfirmed']
 
+# Regular expression and its replacement
+BUG_NO_RE = re.compile(r'\b(bug #?)([0-9])')
+BUG_NO_REPL = r'#\2'
+
 ###
 ### Script begins here
 ###
 
 import os
-import re
 import sys
 import string
 import StringIO
 
 import MySQLdb
 import MySQLdb.cursors
-import trac.env
+try:
+    from trac.env import Environment
+except:
+    from trac.Environment import Environment
 
+from trac import attachment
+
 if not hasattr(sys, 'setdefaultencoding'):
     reload(sys)
 
 sys.setdefaultencoding('latin1')
 
-# simulated Attachment class for trac.add
-class Attachment:
-    def __init__(self, name, data):
-        self.filename = name
-        self.file = StringIO.StringIO(data.tostring())
+    
   
 # simple field translation mapping.  if string not in
 # mapping, just return string, otherwise return value
@@ -134 +202 @@
 
 class TracDatabase(object):
     def __init__(self, path):
-        self.env = trac.env.Environment(path)
+        self.env = Environment(path)
         self._db = self.env.get_db_cnx()
         self._db.autocommit = False
         self.loginNameCache = {}
@@ -210 +278 @@
         c = self.db().cursor()
         c.execute("""DELETE FROM milestone""")
         for ms in m:
-            print "inserting milestone ", ms[key]
+            milestone = ms[key]
+            print "inserting milestone ", milestone
             c.execute("""INSERT INTO milestone (name) VALUES (%s)""",
-                      ms[key].encode('utf-8'))
+                      milestone.encode('utf-8'))
         self.db().commit()
     
     def addTicket(self, id, time, changetime, component,
@@ -226 +295 @@
         if PREFORMAT_COMMENTS:
           desc = '{{{\n%s\n}}}' % desc
 
+        if REPLACE_BUG_NO:
+            if BUG_NO_RE.search(desc):
+                desc = re.sub(BUG_NO_RE, BUG_NO_REPL, desc)
+
         print "inserting ticket %s -- %s" % (id, summary)
         c.execute("""INSERT INTO ticket (id, time, changetime, component,
                                          severity, priority, owner, reporter, cc,
@@ -249 +322 @@
         if PREFORMAT_COMMENTS:
           comment = '{{{\n%s\n}}}' % comment
 
+        if REPLACE_BUG_NO:
+            if BUG_NO_RE.search(comment):
+                comment = re.sub(BUG_NO_RE, BUG_NO_REPL, comment)
+
         c = self.db().cursor()
         c.execute("""INSERT INTO ticket_change (ticket, time, author, field, oldvalue, newvalue)
                                  VALUES        (%s, %s, %s, %s, %s, %s)""",
@@ -261 +338 @@
                                  VALUES        (%s, %s, %s, %s, %s, %s)""",
                   ticket, time.strftime('%s'), author, field, oldvalue.encode('utf-8'), newvalue.encode('utf-8'))
         self.db().commit()
+       
         
-    def addAttachment(self, id, attachment, description, author):
-        print 'inserting attachment for ticket %s -- %s' % (id, description)
-        attachment.filename = attachment.filename.encode('utf-8')
-        self.env.create_attachment(self.db(), 'ticket', str(id), attachment, description.encode('utf-8'),
-            author, 'unknown')
-        
     def getLoginName(self, cursor, userid):
         if userid not in self.loginNameCache:
             cursor.execute("SELECT * FROM profiles WHERE userid = %s" % userid)
@@ -279 +351 @@
                 print 'warning: unknown bugzilla userid %d, recording as anonymous' % userid
                 loginName = 'anonymous'
 
+            loginName = LOGIN_MAP.get(loginName, loginName)
+
             self.loginNameCache[userid] = loginName
 
         return self.loginNameCache[userid]
@@ -298 +372 @@
 
         return self.fieldNameCache[fieldid]
 
-def productFilter(fieldName, products):
-    first = True
-    result = ''
-    for product in products:
-        if not first: 
-            result += " or "
-        first = False
-        result += "%s = '%s'" % (fieldName, product)
-    return result
+def makeWhereClause(fieldName, values, negative=False):
+    if not values:
+        return ''
+    if negative:
+        connector, op = ' AND ', '!='
+    else:
+        connector, op = ' OR ', '='
+    clause = connector.join(["%s %s '%s'" % (fieldName, op, value)
+                             for value in values])
+    return ' WHERE '+clause
 
 def convert(_db, _host, _user, _password, _env, _force):
     activityFields = FieldTranslator()
 
     # account for older versions of bugzilla
     if BZ_VERSION == '2.11':
-        print 'Using Buzvilla v%s schema.' % BZ_VERSION
+        print 'Using Bugzilla v%s schema.' % BZ_VERSION
         activityFields['removed'] = 'oldvalue'
         activityFields['added'] = 'newvalue'
 
@@ -337 +412 @@
         c.execute("""DELETE FROM ticket""")
         trac.db().commit()
         c.execute("""DELETE FROM attachment""")
-        os.system('rm -rf %s' % trac.env.get_attachments_dir())
-        os.mkdir(trac.env.get_attachments_dir())
+        attachments_dir = os.path.join(os.path.normpath(trac.env.path), 'attachments')
+        os.system('rm -rf %s' % attachments_dir)
+        os.mkdir(attachments_dir)
         trac.db().commit()
 
 
     print
+    print "0. filtering products..."
+    sql = "SELECT product FROM products"
+    mysql_cur.execute(sql)
+    products = []
+    for line in mysql_cur.fetchall():
+        product = line['product']
+        if PRODUCTS and product not in PRODUCTS:
+            continue
+        if product in IGNORE_PRODUCTS:
+            continue
+        products.append(product)
+    PRODUCTS[:] = products
+    print "using products", ' '.join(PRODUCTS)
+
+    print
     print "1. import severities..."
-    severities = (('blocker', '1'), ('critical', '2'), ('major', '3'), ('normal', '4'),
-        ('minor', '5'), ('trivial', '6'), ('enhancement', '7'))
-    trac.setSeverityList(severities)
+    trac.setSeverityList(SEVERITIES)
 
     print
     print "2. import components..."
-    sql = "SELECT value, initialowner AS owner FROM components"
-    if PRODUCTS:
-       sql += " WHERE %s" % productFilter('program', PRODUCTS)
-    mysql_cur.execute(sql)
-    components = mysql_cur.fetchall()
-    for component in components:
-                component['owner'] = trac.getLoginName(mysql_cur, component['owner'])
-    trac.setComponentList(components, 'value')
+    if not COMPONENTS_FROM_PRODUCTS:
+        sql = "SELECT value, initialowner AS owner FROM components"
+        sql += makeWhereClause('program', PRODUCTS)
+        mysql_cur.execute(sql)
+        components = mysql_cur.fetchall()
+        for component in components:
+            component['owner'] = trac.getLoginName(mysql_cur, component['owner'])
+        trac.setComponentList(components, 'value')
+    else:
+        sql = "SELECT program AS product, value AS comp, initialowner AS owner FROM components"
+        sql += makeWhereClause('program', PRODUCTS)
+        mysql_cur.execute(sql)
+        lines = mysql_cur.fetchall()
+        all_components = {} # product -> components
+        all_owners = {} # product, component -> owner
+        for line in lines:
+            product = line['product']
+            comp = line['comp']
+            owner = line['owner']
+            all_components.setdefault(product, []).append(comp)
+            all_owners[(product, comp)] = owner
+        component_list = []
+        for product, components in all_components.items():
+            # find best default owner
+            default = None
+            for comp in DEFAULT_COMPONENTS:
+                if comp in components:
+                    default = comp
+                    break
+            if default is None:
+                default = components[0]
+            owner = all_owners[(product, default)]
+            owner_name = trac.getLoginName(mysql_cur, owner)
+            component_list.append({'product': product, 'owner': owner_name})
+        trac.setComponentList(component_list, 'product')
 
     print
     print "3. import priorities..."
-    priorities = (('P1', '1'), ('P2', '2'), ('P3', '3'), ('P4', '4'), ('P5', '5'))
-    trac.setPriorityList(priorities)
+    trac.setPriorityList(PRIORITIES)
 
     print
     print "4. import versions..."
     sql = "SELECT DISTINCTROW value FROM versions"
-    if PRODUCTS:
-       sql += " WHERE %s" % productFilter('program', PRODUCTS)
+    sql += makeWhereClause('program', PRODUCTS)
     mysql_cur.execute(sql)
     versions = mysql_cur.fetchall()
     trac.setVersionList(versions, 'value')
 
     print
     print "5. import milestones..."
-    mysql_cur.execute("SELECT value FROM milestones")
+    sql = "SELECT DISTINCT value FROM milestones"
+    sql += makeWhereClause('value', IGNORE_MILESTONES, negative=True)
+    mysql_cur.execute(sql)
     milestones = mysql_cur.fetchall()
-    if milestones[0] == '---':
-        trac.setMilestoneList(milestones, 'value')
-    else:
-        trac.setMilestoneList([], '')
+    trac.setMilestoneList(milestones, 'value')
 
     print
     print '6. retrieving bugs...'
     sql = "SELECT * FROM bugs "
-    if PRODUCTS:
-       sql += " WHERE %s" % productFilter('product', PRODUCTS)
+    sql += makeWhereClause('product', PRODUCTS)
     sql += " ORDER BY bug_id"
     mysql_cur.execute(sql)
     bugs = mysql_cur.fetchall()
@@ -401 +513 @@
         ticket['id'] = bugid
         ticket['time'] = bug['creation_ts']
         ticket['changetime'] = bug['delta_ts']
-        ticket['component'] = bug['component']
+        if COMPONENTS_FROM_PRODUCTS:
+            ticket['component'] = bug['product']
+        else:
+            ticket['component'] = bug['component']
         ticket['severity'] = bug['bug_severity']
         ticket['priority'] = bug['priority']
 
@@ -413 +528 @@
         cc_list = []
         for cc in cc_records:
             cc_list.append(trac.getLoginName(mysql_cur, cc['who']))
+        cc_list = [cc for cc in cc_list if '@' in cc and cc not in IGNORE_CC]
         ticket['cc'] = string.join(cc_list, ', ')
 
         ticket['version'] = bug['version']
 
-        if bug['target_milestone'] == '---':
-            ticket['milestone'] = ''
-        else:
-            ticket['milestone'] = bug['target_milestone']
+        target_milestone = bug['target_milestone']
+        if target_milestone in IGNORE_MILESTONES:
+            target_milestone = ''
+        ticket['milestone'] = target_milestone
 
         bug_status = bug['bug_status'].lower()
         ticket['status'] = statusXlator[bug_status]
@@ -435 +551 @@
 
         ticket['summary'] = bug['short_desc']
 
-        keywords = string.split(bug['keywords'], ' ')
-
         mysql_cur.execute("SELECT * FROM longdescs WHERE bug_id = %s" % bugid) 
         longdescs = list(mysql_cur.fetchall())
 
@@ -464 +578 @@
         bugs_activity = mysql_cur.fetchall()
         resolution = ''
         ticketChanges = []
+        keywords = []
         for activity in bugs_activity:
             field_name = trac.getFieldName(mysql_cur, activity['fieldid']).lower()
             
@@ -479 +594 @@
             if field_name == 'resolution':
                 resolution = added.lower()
 
-            keywordChange = False
-            oldKeywords = string.join(keywords, " ")
+            add_keywords = []
+            remove_keywords = []
 
             # convert bugzilla field names...
             if field_name == 'bug_severity':
@@ -490 +605 @@
             elif field_name == 'bug_status':
                 field_name = 'status'
                 if removed in STATUS_KEYWORDS:
-                    kw = STATUS_KEYWORDS[removed]
-                    if kw in keywords:
-                        keywords.remove(kw)
-                    else:
-                        oldKeywords = string.join(keywords + [ kw ], " ")
-                    keywordChange = True
+                    remove_keywords.append(STATUS_KEYWORDS[removed])
                 if added in STATUS_KEYWORDS:
-                    kw = STATUS_KEYWORDS[added]
-                    keywords.append(kw)
-                    keywordChange = True
+                    add_keywords.append(STATUS_KEYWORDS[added])
                 added = statusXlator[added]
                 removed = statusXlator[removed]
             elif field_name == 'short_desc':
                 field_name = 'summary'
-            elif field_name == 'product':
-                if removed in PRODUCT_KEYWORDS:
-                    kw = PRODUCT_KEYWORDS[removed]
-                    if kw in keywords:
-                        keywords.remove(kw)
-                    else:
-                        oldKeywords = string.join(keywords + [ kw ], " ")
-                    keywordChange = True
-                if added in PRODUCT_KEYWORDS:
-                    kw = PRODUCT_KEYWORDS[added]
-                    keywords.append(kw)
-                    keywordChange = True
+            elif field_name == 'product' and COMPONENTS_FROM_PRODUCTS:
+                field_name = 'component'
+            elif ((field_name == 'product' and not COMPONENTS_FROM_PRODUCTS) or
+                  (field_name == 'component' and COMPONENTS_FROM_PRODUCTS)):
+                if MAP_ALL_KEYWORDS or removed in KEYWORDS_MAPPING:
+                    kw = KEYWORDS_MAPPING.get(removed, removed)
+                    if kw:
+                        remove_keywords.append(kw)
+                if MAP_ALL_KEYWORDS or added in KEYWORDS_MAPPING:
+                    kw = KEYWORDS_MAPPING.get(added, added)
+                    if kw:
+                        add_keywords.append(kw)
+                if field_name == 'component':
+                    # just keep the keyword change
+                    added = removed = ''
+            elif field_name == 'target_milestone':
+                field_name = 'milestone'
+                if added in IGNORE_MILESTONES:
+                    added = ''
+                if removed in IGNORE_MILESTONES:
+                    removed = ''
 
             ticketChange = {}
             ticketChange['ticket'] = bugid
@@ -525 +642 @@
             ticketChange['oldvalue'] = removed
             ticketChange['newvalue'] = added
 
-            if keywordChange:
-                newKeywords = string.join(keywords, " ")
-                ticketChangeKw = ticketChange
-                ticketChangeKw['field'] = 'keywords'
-                ticketChangeKw['oldvalue'] = oldKeywords
-                ticketChangeKw['newvalue'] = newKeywords
-                #trac.addTicketChange(ticket=bugid, time=activity['bug_when'],
-                #    author=trac.getLoginName(mysql_cur, activity['who']),
-                #    field='keywords', oldvalue=oldKeywords, newvalue=newKeywords)
-                ticketChanges.append(ticketChangeKw)
+            if add_keywords or remove_keywords:
+                # ensure removed ones are in old
+                old_keywords = keywords + [kw for kw in remove_keywords if kw not in keywords]
+                # remove from new
+                keywords = [kw for kw in keywords if kw not in remove_keywords]
+                # add to new
+                keywords += [kw for kw in add_keywords if kw not in keywords]
+                if old_keywords != keywords:
+                    ticketChangeKw = ticketChange.copy()
+                    ticketChangeKw['field'] = 'keywords'
+                    ticketChangeKw['oldvalue'] = ' '.join(old_keywords)
+                    ticketChangeKw['newvalue'] = ' '.join(keywords)
+                    ticketChanges.append(ticketChangeKw)
 
             if field_name in IGNORED_ACTIVITY_FIELDS:
                 continue
@@ -552 +672 @@
                   oldChange['oldvalue'] += " " + ticketChange['oldvalue'] 
                   oldChange['newvalue'] += " " + ticketChange['newvalue']
                   break
+              # cc sometime appear in different activities with same time
+              if (field_name == 'cc'
+                  and oldChange['time'] == ticketChange['time']):
+                  oldChange['newvalue'] += ", " + ticketChange['newvalue']
+                  break
             else:
-                #trac.addTicketChange(ticket=bugid, time=activity['bug_when'],
-                #    author=trac.getLoginName(mysql_cur, activity['who']),
-                #    field=field_name, oldvalue=removed, newvalue=added)
                 ticketChanges.append (ticketChange)
 
         for ticketChange in ticketChanges:
@@ -568 +690 @@
         if not ticket['resolution'] and ticket['status'] == 'closed':
             ticket['resolution'] = resolution
 
-        if bug['bug_status'] in STATUS_KEYWORDS:
-            kw = STATUS_KEYWORDS[bug['bug_status']]
-            # may have already been added during activity import
+        bug_status = bug['bug_status']
+        if bug_status in STATUS_KEYWORDS:
+            kw = STATUS_KEYWORDS[bug_status]
             if kw not in keywords:
                 keywords.append(kw)
 
-        if bug['product'] in PRODUCT_KEYWORDS:
-            kw = PRODUCT_KEYWORDS[bug['product']]
-            # may have already been added during activity import
-            if kw not in keywords:
+        product = bug['product']
+        if product in KEYWORDS_MAPPING and not COMPONENTS_FROM_PRODUCTS:
+            kw = KEYWORDS_MAPPING.get(product, product)
+            if kw and kw not in keywords:
                 keywords.append(kw)
 
+        component = bug['component']
+        if (COMPONENTS_FROM_PRODUCTS and
+            (MAP_ALL_KEYWORDS or component in KEYWORDS_MAPPING)):
+            kw = KEYWORDS_MAPPING.get(component, component)
+            if kw and kw not in keywords:
+                keywords.append(kw)
+
         mysql_cur.execute("SELECT * FROM attachments WHERE bug_id = %s" % bugid)
         attachments = mysql_cur.fetchall()
         for a in attachments:
             author = trac.getLoginName(mysql_cur, a['submitter_id'])
             
-            tracAttachment = Attachment(a['filename'], a['thedata'])
-            trac.addAttachment(bugid, tracAttachment, a['description'], author)
+            #tracAttachment = Attachment(a['filename'], a['thedata'])
+            #trac.addAttachment(bugid, tracAttachment, a['description'], author)
+            #self.env.create_attachment(self.db(), 'ticket', str(id), attachment, description.encode('utf-8'),
+
+            tracAttachment = attachment.Attachment(trac.env, 'ticket', str(a['bug_id']), None, trac.db())
+            tracAttachment.insert(a['filename'], StringIO.StringIO(a['thedata'].tostring()), len(a['thedata']))
             
         ticket['keywords'] = string.join(keywords)                
         ticketid = trac.addTicket(**ticket)
@@ -639 +772 @@
                 print "Error: unknown parameter: " + sys.argv[iter]
                 sys.exit(0)
             iter = iter + 1
-    else:
-        usage()
         
     convert(BZ_DB, BZ_HOST, BZ_USER, BZ_PASSWORD, TRAC_ENV, TRAC_CLEAN)