Edgewall Software

source: branches/1.4-stable/contrib/sourceforge2trac.py

Last change on this file was 17656, checked in by Jun Omae, 8 months ago

1.4.4dev: update copyright year to 2023 (refs #13476)

[skip ci]
  • Property svn:eol-style set to native
  • Property svn:executable set to *
File size: 24.7 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#
4# Copyright (C) 2004-2023 Edgewall Software
5# Copyright (C) 2004 Dmitry Yusupov <dmitry_yus@yahoo.com>
6# Copyright (C) 2004 Mark Rowe <mrowe@bluewire.net.nz>
7# Copyright (C) 2010 Anatoly Techtonik <techtonik@php.net>
8# All rights reserved.
9#
10# This software is licensed as described in the file COPYING, which
11# you should have received as part of this distribution. The terms
12# are also available at https://trac.edgewall.org/wiki/TracLicense.
13#
14# This software consists of voluntary contributions made by many
15# individuals. For the exact contribution history, see the revision
16# history and logs, available at https://trac.edgewall.org/.
17
18"""
19Import a Sourceforge project's tracker items into a Trac database.
20
21Requires:
22 Trac 1.0 from https://trac.edgewall.org/
23 Python 2.5 from http://www.python.org/
24
251.0 clean-up by cboos **untested**, use at your own risks and send patches
26
27The Sourceforge tracker items can be exported from the 'Backup' page
28of the project admin section. Substitute XXXXX with project id:
29https://sourceforge.net/export/xml_export2.php?group_id=XXXXX
30
31$Id$
32
33
34Uses Trac 0.11 DB format version 21
35SourceForge XML Export format identified by the header:
36<!DOCTYPE project_export SYSTEM "http://sourceforge.net/export/sf_project_export_0.2.dtd">
37
38Works with all DB backends. Attachments are not downloaded, but inserted
39as links to SF tracker.
40
41
42Ticket Types, Priorities and Resolutions
43----------------------------------------
44Conversion kills default Trac ticket types:
45- defect 1
46- enhancement 2
47- task 3
48
49and priorities:
50- blocker 1
51- critical 2
52- major 3
53- minor 4
54- trivial 5
55
56and resolutions:
57- fixed 1
58- invalid 2
59- wontfix 3
60- duplicate 4
61- worksforme 5
62
63
64Versions and Milestones
65-----------------------
66Kills versions and milestones from existing Trac DB
67
68
69Mapping
70-------
71tracker_name == ticket_type
72group_name == version
73category_name == component
74
75user nobody == anonymous
76
77
78Not implemented (feature:reason)
79--------------------------------
80attachments:made as a comment with links to attachments stored on SF
81 (type,id,filename,size,time,description,author)
82ticket_custom:unknown (ticket,name,value)
83history:imported only for summary, priority. closed date and owner fields
84
85severities:no field in source data
86"""
87
88import argparse
89import sys
90import time
91from xml.etree.ElementTree import ElementTree
92
93from trac.env import Environment
94from trac.util.text import printerr
95
96#: rename users from SF to Trac
97user_map = {"nobody":"anonymous"}
98
99complete_msg = """
100Conversion complete.
101
102You may want to login into Trac to verify names for ticket owners. You may
103also want to rename ticket types and priorities to default.
104"""
105
106
107# --- utility
108class DBNotEmpty(Exception):
109 def __str__(self):
110 return "Will not modify database with existing tickets!"
111
112class FlatXML(object):
113 """Flat XML is XML without element attributes. Also each element
114 may contain other elements or text, but not both.
115
116 This object mirrors XML structure into own properties for convenient
117 access to tree elements, i.e. flat.trackers[2].groups[2].group_name
118
119 Uses recursion.
120 """
121
122 def __init__(self, el=None):
123 """el is ElementTree element"""
124 if el:
125 self.merge(el)
126
127 def merge(self, el):
128 """merge supplied ElementTree element into current object"""
129 for c in el:
130 if len(c.getchildren()) == 0:
131 if c.text is not None and len(c.text.strip()) != 0:
132 self.__setattr__(c.tag, c.text)
133 else:
134 self.__setattr__(c.tag, [])
135 else: #if c.getchildren()[0].tag == c.tag[:-1]:
136 # c is a set of elements
137 self.__setattr__(c.tag, [FlatXML(x) for x in c.getchildren()])
138
139
140 def __str__(self):
141 buf = ""
142 for sub in self.__dict__:
143 val = self.__dict__[sub]
144 if type(val) != list:
145 buf += "%s : %s\n" % (sub, val)
146 else:
147 for x in val:
148 buf += "\n ".join(x.__str__().split("\n"))
149 return buf
150
151 def __repr__(self):
152 buf = ""
153 for sub in self.__dict__:
154 val = self.__dict__[sub]
155 if type(val) != list:
156 buf += "<%s>%s</%s>\n" % (sub, val, sub)
157 else:
158 for x in val:
159 buf += "\n ".join(x.__repr__().split("\n"))
160 return buf
161
162
163# --- SF data model
164class Tracker(FlatXML):
165 """
166 <trackers>
167 <tracker>
168 <url>http://sourceforge.net/?group_id=175454&#38;atid=873299</url>
169 <tracker_id>873299</tracker_id>
170 <name>Bugs</name>
171 <description>Bug Tracking System</description>
172 <is_public>All site users</is_public>
173 <allow_anon>Yes</allow_anon>
174 <email_updates>Send to goblinhack@gmail.com</email_updates>
175 <due_period>2592000</due_period>
176 <submit_instructions></submit_instructions>
177 <browse_instructions></browse_instructions>
178 <status_timeout>1209600</status_timeout>
179 <due_period_initial>0</due_period_initial>
180 <due_period_update>0</due_period_update>
181 <reopen_on_comment>1</reopen_on_comment>
182 <canned_responses>
183 </canned_responses>
184 <groups>
185 <group>
186 <id>632324</id>
187 <group_name>v1.0 (example)</group_name>
188 </group>
189 </groups>
190 <categories>
191 <category>
192 <id>885178</id>
193 <category_name>Interface (example)</category_name>
194 <auto_assignee>nobody</auto_assignee>
195 </category>
196 </categories>
197 <resolutions>
198 <resolution>
199 <id>1</id>
200 <name>Fixed</name>
201 </resolution>
202 <resolution>
203 <id>2</id>
204 <name>Invalid</name>
205 </resolution>
206 ...
207 </resolutions>
208 <statuses>
209 <status>
210 <id>1</id>
211 <name>Open</name>
212 </status>
213 <status>
214 <id>2</id>
215 <name>Closed</name>
216 </status>
217 <status>
218 <id>3</id>
219 <name>Deleted</name>
220 </status>
221 <status>
222 <id>4</id>
223 <name>Pending</name>
224 </status>
225 </statuses>
226 ...
227 <tracker_items>
228 <tracker_item>
229<url>http://sourceforge.net/support/tracker.php?aid=2471428</url>
230<id>2471428</id>
231<status_id>2</status_id>
232<category_id>100</category_id>
233<group_id>100</group_id>
234<resolution_id>100</resolution_id>
235<submitter>sbluen</submitter>
236<assignee>nobody</assignee>
237<closer>goblinhack</closer>
238<submit_date>1230400444</submit_date>
239<close_date>1231087612</close_date>
240<priority>5</priority>
241<summary>glitch with edge of level</summary>
242<details>The mini-laser that the future soldier carries is so powerful that it even lets me go outside the level. I stand at the top edge of the level and then shoot up, and then it gets me somewhere where I am not supposed to go.</details>
243<is_private>0</is_private>
244<followups>
245 <followup>
246 <id>2335316</id>
247 <submitter>goblinhack</submitter>
248 <date>1175610236</date>
249 <details>Logged In: YES
250 user_id=1577972
251 Originator: NO
252
253 does this happen every game or just once?
254
255 you could send me the saved file and I'll try and load it - old
256 versions harldy ever work with newer versions - need to add some
257 kind of warnings on that
258
259 tx</details>
260 </followup>
261 ...
262</followups>
263<attachments>
264 <attachment>
265 <url>http://sourceforge.net/tracker/download.php?group_id=175454&#38;atid=873299&#38;file_id=289080&#38;aid=</url>
266 <id>289080</id>
267 <filename>your_most_recent_game.gz</filename>
268 <description>my saved game</description>
269 <filesize>112968</filesize>
270 <filetype>application/x-gzip</filetype>
271 <date>1218987770</date>
272 <submitter>sbluen</submitter>
273 </attachment>
274...
275</attachments>
276<history_entries>
277 <history_entry>
278 <id>7304242</id>
279 <field_name>IP</field_name>
280 <old_value>Artifact Created: 76.173.48.148</old_value>
281 <date>1230400444</date>
282 <updator>sbluen</updator>
283 </history_entry>
284 ...
285</history_entries>
286 </tracker_item>
287 ...
288 </tracker_items>
289 ...
290 </tracker>
291 </trackers>
292 """
293 def __init__(self, e):
294 self.merge(e)
295
296
297class ExportedProjectData(object):
298 """Project data container as Python object.
299 """
300 def __init__(self, f):
301 """Data parsing"""
302
303 self.trackers = [] #: tracker properties and data
304 self.groups = [] #: groups []
305 self.priorities = [] #: priorities used
306 self.resolutions = [] #: resolutions (index, name)
307 self.tickets = [] #: all tickets
308 self.statuses = [] #: status (idx, name)
309
310 self.used_resolutions = {} #: id:name
311 self.used_categories = {} #: id:name
312 # id '100' means no category
313 self.used_categories['100'] = None
314 self.users = {} #: id:name
315
316 root = ElementTree().parse(f)
317
318 self.users = {FlatXML(u).userid: FlatXML(u).username
319 for u in root.find('referenced_users')}
320
321 for tracker in root.find('trackers'):
322 tr = Tracker(tracker)
323 self.trackers.append(tr)
324
325 # groups-versions
326 for grp in tr.groups:
327 # group ids are tracker-specific even if names match
328 g = (grp.id, grp.group_name)
329 if g not in self.groups:
330 self.groups.append(g)
331
332 # resolutions
333 for res in tr.resolutions:
334 r = (res.id, res.name)
335 if r not in self.resolutions:
336 self.resolutions.append(r)
337
338 # statuses
339 self.statuses = [(s.id, s.name) for s in tr.statuses]
340
341 # tickets
342 for tck in tr.tracker_items:
343 if type(tck) == str:
344 print(repr(tck))
345 self.tickets.append(tck)
346 if int(tck.priority) not in self.priorities:
347 self.priorities.append(int(tck.priority))
348 res_id = getattr(tck, "resolution_id", None)
349 if res_id is not None and res_id not in self.used_resolutions:
350 for idx, name in self.resolutions:
351 if idx == res_id: break
352 self.used_resolutions[res_id] = \
353 dict(self.resolutions)[res_id]
354 # used categories
355 categories = dict(self.get_categories(tr, noowner=True))
356 if tck.category_id not in self.used_categories:
357 self.used_categories[tck.category_id] = \
358 categories[tck.category_id]
359
360 # sorting everything
361 self.trackers.sort(key=lambda x:x.name)
362 self.groups.sort()
363 self.priorities.sort()
364
365 def get_categories(self, tracker=None, noid=False, noowner=False):
366 """ SF categories : Trac components
367 (id, name, owner) tuples for specified tracker or all trackers
368 if noid or noowner flags are set, specified tuple attribute is
369 stripped
370 """
371 trs = [tracker] if tracker is not None else self.trackers
372 categories = []
373 for tr in trs:
374 for cat in tr.categories:
375 c = (cat.id, cat.category_name, cat.auto_assignee)
376 if c not in categories:
377 categories.append(c)
378 #: sort by name
379 if noid:
380 categories.sort()
381 else:
382 categories.sort(key=lambda x:x[1])
383 if noowner:
384 categories = [x[:2] for x in categories]
385 if noid:
386 categories = [x[1:] for x in categories]
387 return categories
388
389
390class TracDatabase(object):
391 def __init__(self, path):
392 self.env = Environment(path)
393
394 def hasTickets(self):
395 return int(self.env.db_query("SELECT count(*) FROM ticket")[0][0]) > 0
396
397 def dbCheck(self):
398 if self.hasTickets():
399 raise DBNotEmpty
400
401 def setTypeList(self, s):
402 """Remove all types, set them to `s`"""
403 self.dbCheck()
404 with self.env.db_transaction as db:
405 db("DELETE FROM enum WHERE type='ticket_type'")
406 for i, value in enumerate(s):
407 db("INSERT INTO enum (type, name, value) VALUES (%s, %s, %s)",
408 ("ticket_type", value, i))
409
410 def setPriorityList(self, s):
411 """Remove all priorities, set them to `s`"""
412 self.dbCheck()
413 with self.env.db_transaction as db:
414 db("DELETE FROM enum WHERE type='priority'")
415 for i, value in enumerate(s):
416 db("INSERT INTO enum (type, name, value) VALUES (%s, %s, %s)",
417 ("priority", value, i))
418
419 def setResolutionList(self, t):
420 """Remove all resolutions, set them to `t` (index, name)"""
421 self.dbCheck()
422 with self.env.db_transaction as db:
423 db("DELETE FROM enum WHERE type='resolution'")
424 for value, name in t:
425 db("INSERT INTO enum (type, name, value) VALUES (%s, %s, %s)",
426 ("resolution", name, value))
427
428 def setComponentList(self, t):
429 """Remove all components, set them to `t` (name, owner)"""
430 self.dbCheck()
431 with self.env.db_transaction as db:
432 db("DELETE FROM component")
433 for name, owner in t:
434 db("INSERT INTO component (name, owner) VALUES (%s, %s)",
435 (name, owner))
436
437 def setVersionList(self, v):
438 """Remove all versions, set them to `v`"""
439 self.dbCheck()
440 with self.env.db_transaction as db:
441 db("DELETE FROM version")
442 for value in v:
443 # time and description are also available
444 db("INSERT INTO version (name) VALUES (%s)", value)
445
446 def setMilestoneList(self, m):
447 """Remove all milestones, set them to `m` ("""
448 self.dbCheck()
449 with self.env.db_transaction as db:
450 db("DELETE FROM milestone")
451 for value in m:
452 # due, completed, description are also available
453 db("INSERT INTO milestone (name) VALUES (%s)", value)
454
455 def addTicket(self, type, time, changetime, component,
456 priority, owner, reporter, cc,
457 version, milestone, status, resolution,
458 summary, description, keywords):
459 """ ticket table db21.py format
460
461 id integer PRIMARY KEY,
462 type text, -- the nature of the ticket
463 time integer, -- the time it was created
464 changetime integer,
465 component text,
466 severity text,
467 priority text,
468 owner text, -- who is this ticket assigned to
469 reporter text,
470 cc text, -- email addresses to notify
471 version text, --
472 milestone text, --
473 status text,
474 resolution text,
475 summary text, -- one-line summary
476 description text, -- problem description (long)
477 keywords text
478 """
479 if status.lower() == 'open':
480 if owner != '':
481 status = 'assigned'
482 else:
483 status = 'new'
484
485 with self.env.db_transaction as db:
486 c = db.cursor()
487 c.execute("""
488 INSERT INTO ticket (type, time, changetime, component,
489 priority, owner, reporter, cc, version,
490 milestone, status, resolution, summary,
491 description, keywords)
492 VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
493 %s, %s)
494 """, (type, time, changetime, component, priority, owner,
495 reporter, cc, version, milestone, status.lower(),
496 resolution, summary, '%s' % description, keywords))
497 return db.get_last_id(c, 'ticket')
498
499 def addTicketComment(self, ticket, time, author, value):
500 with self.env.db_transaction as db:
501 db("""
502 INSERT INTO ticket_change (ticket, time, author, field,
503 oldvalue, newvalue)
504 VALUES (%s, %s, %s, %s, %s, %s)
505 """, (ticket, time, author, 'comment', '', '%s' % value))
506
507 def addTicketChange(self, ticket, time, author, field, oldvalue, newvalue):
508 with self.env.db_transaction as db:
509 db("""INSERT INTO ticket_change (ticket, time, author, field,
510 oldvalue, newvalue)
511 VALUES (%s, %s, %s, %s, %s, %s)
512 """, (ticket, time, author, field, oldvalue, newvalue))
513
514
515def importData(f, env):
516 project = ExportedProjectData(f)
517 trackers = project.trackers
518
519 trac = TracDatabase(env)
520
521 # Data conversion
522 typeList = [x.name for x in trackers]
523 print("%d trackers will be converted to the following ticket types:\n %s"
524 % (len(trackers), typeList))
525
526 used_cat_names = set(project.used_categories.values())
527 #: make names unique, forget about competing owners (the last one wins)
528 components = dict(project.get_categories(noid=True)).items()
529 components.sort()
530 components = [x for x in components if x[0] in used_cat_names]
531 print("%d out of %d categories are used and will be converted to the"
532 " following components:\n %s"
533 % (len(components), len(project.get_categories()), components))
534 print("..renaming component owners:")
535 for i,c in enumerate(components):
536 if c[1] in user_map:
537 components[i] = (c[0], user_map[c[1]])
538 print(" %s" % components)
539
540 print("%d groups which will be converted to the following versions:\n"
541 " %s" % (len(project.groups), project.groups))
542 print("%d resolutions found :\n %s"
543 % (len(project.resolutions), project.resolutions))
544 resolutions = [(k,project.used_resolutions[k])
545 for k in project.used_resolutions]
546 resolutions.sort(key=lambda x:int(x[0]))
547 print(".. only %d used will be imported:\n %s"
548 % (len(resolutions), resolutions))
549 print("Priorities used so far: %s" % project.priorities)
550 if not(raw_input("Continue [y/N]?").lower() == 'y'):
551 sys.exit()
552
553 # Data save
554 trac.setTypeList(typeList)
555 trac.setComponentList(components)
556 trac.setPriorityList(xrange(min(project.priorities),
557 max(project.priorities)))
558 trac.setVersionList({x[1] for x in project.groups})
559 trac.setResolutionList(resolutions)
560 trac.setMilestoneList([])
561
562 for tracker in project.trackers:
563 # id 100 means no component selected
564 component_lookup = dict(project.get_categories(noowner=True) +
565 [("100", None)])
566 for t in tracker.tracker_items:
567 i = trac.addTicket(type=tracker.name,
568 time=int(t.submit_date),
569 changetime=int(t.submit_date),
570 component=component_lookup[t.category_id],
571 priority=t.priority,
572 owner=t.assignee \
573 if t.assignee not in user_map \
574 else user_map[t.assignee],
575 reporter=t.submitter \
576 if t.submitter not in user_map \
577 else user_map[t.submitter],
578 cc=None,
579 # 100 means no group selected
580 version=dict(project.groups +
581 [("100", None)])[t.group_id],
582 milestone=None,
583 status=dict(project.statuses)[t.status_id],
584 resolution=dict(resolutions)[t.resolution_id] \
585 if hasattr(t, "resolution_id") else None,
586 summary=t.summary,
587 description=t.details,
588 keywords='sf' + t.id)
589
590 print("Imported %s as #%d" % (t.id, i))
591
592 if len(t.attachments):
593 attmsg = "SourceForge attachments:\n"
594 for a in t.attachments:
595 attmsg = attmsg + " * [%s %s] (%s) - added by '%s' %s [[BR]] "\
596 % (a.url+t.id, a.filename, a.filesize+" bytes",
597 user_map.get(a.submitter, a.submitter),
598 time.strftime("%Y-%m-%d %H:%M:%S",
599 time.localtime(int(a.date))))
600 attmsg = attmsg + "''%s ''\n" % (a.description or '')
601 # empty description is as empty list
602 trac.addTicketComment(ticket=i,
603 time=time.strftime("%Y-%m-%d %H:%M:%S",
604 time.localtime(int(t.submit_date))),
605 author=None, value=attmsg)
606 print(" added information about %d attachments for #%d"
607 % (len(t.attachments), i))
608
609 for msg in t.followups:
610 """
611 <followup>
612 <id>3280792</id>
613 <submitter>goblinhack</submitter>
614 <date>1231087739</date>
615 <details>done</details>
616 </followup>
617 """
618 trac.addTicketComment(ticket=i,
619 time=msg.date,
620 author=msg.submitter,
621 value=msg.details)
622 if t.followups:
623 print(" imported %d messages for #%d"
624 % (len(t.followups), i))
625
626 # Import history
627 """
628 <history_entry>
629 <id>4452195</id>
630 <field_name>resolution_id</field_name>
631 <old_value>100</old_value>
632 <date>1176043865</date>
633 <updator>goblinhack</updator>
634 </history_entry>
635 """
636 revision = t.__dict__.copy()
637
638 # iterate the history in reverse order and update ticket revision from
639 # current (last) to initial
640 changes = 0
641 for h in sorted(t.history_entries, reverse=True):
642 """
643 Processed fields (field - notes):
644 IP - no target field, just skip
645 summary
646 priority
647 close_date
648 assigned_to
649
650 Fields not processed (field: explanation):
651 File Added - TODO
652 resolution_id - need to update used_resolutions
653 status_id
654 artifact_group_id
655 category_id
656 group_id
657 """
658 f = None
659 if h.field_name in ("IP",):
660 changes += 1
661 continue
662 elif h.field_name in ("summary", "priority"):
663 f = h.field_name
664 oldvalue = h.old_value
665 newvalue = revision.get(h.field_name)
666 elif h.field_name == 'assigned_to':
667 f = "owner"
668 newvalue = revision['assignee']
669 if h.old_value == '100': # was not assigned
670 revision['assignee'] = None
671 oldvalue = None
672 else:
673 username = project.users[h.old_value]
674 if username in user_map: username = user_map[username]
675 revision['assignee'] = oldvalue = username
676 elif h.field_name == 'close_date' and revision['close_date'] != 0:
677 f = 'status'
678 oldvalue = 'assigned'
679 newvalue = 'closed'
680
681 if f:
682 changes += 1
683 trac.addTicketChange(ticket=i,
684 time=h.date,
685 author=h.updator,
686 field=f,
687 oldvalue=oldvalue,
688 newvalue=newvalue)
689
690 if h.field_name != 'assigned_to':
691 revision[h.field_name] = h.old_value
692 if changes:
693 print(" processed %d out of %d history items for #%d"
694 % (changes, len(t.history_entries), i))
695
696
697def main():
698 parser = argparse.ArgumentParser()
699 parser.add_argument('filename', help="XML export file from sourceforge")
700 parser.add_argument('environment', help="path to Trac environment")
701 args = parser.parse_args()
702
703 try:
704 with open(args.filename, 'r') as f:
705 importData(f, args.environment)
706 except Exception as e:
707 printerr("Error: %s" % e)
708 sys.exit(1)
709
710 print(complete_msg)
711
712
713if __name__ == '__main__':
714 main()
Note: See TracBrowser for help on using the repository browser.