migrate_from_mantis.rake

modified migration script KH - Klaus Heerlein, 2021-02-19 12:36

Download (21.7 KB)

 
1
# Redmine - project management software
2
# Copyright (C) 2006-2017  Jean-Philippe Lang
3
#
4
# This program is free software; you can redistribute it and/or
5
# modify it under the terms of the GNU General Public License
6
# as published by the Free Software Foundation; either version 2
7
# of the License, or (at your option) any later version.
8
#
9
# This program is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
# GNU General Public License for more details.
13
#
14
# You should have received a copy of the GNU General Public License
15
# along with this program; if not, write to the Free Software
16
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
17

    
18
# K. Heerlein 2021-02-18
19
# modified to be able to import MANTIS 2.24.4 data to Redmine  4.0.6.stable
20
# quick and dirty mods, no warranty !!
21
# added some commenting to understand what is going on  
22
desc 'Mantis migration script'
23

    
24
require 'active_record'
25
require 'pp'
26

    
27
namespace :redmine do
28
task :migrate_from_mantis => :environment do
29

    
30
  module MantisMigrate
31

    
32
      new_status = IssueStatus.find_by_position(1)
33
      assigned_status = IssueStatus.find_by_position(2)
34
      resolved_status = IssueStatus.find_by_position(3)
35
      feedback_status = IssueStatus.find_by_position(4)
36
      closed_status = IssueStatus.where(:is_closed => true).first
37
      STATUS_MAPPING = {10 => new_status,      # new
38
                        20 => feedback_status, # feedback
39
                        30 => new_status,      # acknowledged
40
                        40 => new_status,      # confirmed
41
                        50 => assigned_status, # assigned
42
                        80 => resolved_status, # resolved
43
                        90 => closed_status    # closed
44
                        }
45

    
46
      priorities = IssuePriority.all
47
      DEFAULT_PRIORITY = priorities[2]
48
      PRIORITY_MAPPING = {10 => priorities[1], # none
49
                          20 => priorities[1], # low
50
                          30 => priorities[2], # normal
51
                          40 => priorities[3], # high
52
                          50 => priorities[4], # urgent
53
                          60 => priorities[5]  # immediate
54
                          }
55

    
56
      TRACKER_BUG = Tracker.find_by_position(1)
57
      TRACKER_FEATURE = Tracker.find_by_position(2)
58

    
59
      roles = Role.where(:builtin => 0).order('position ASC').all
60
      manager_role = roles[0]
61
      developer_role = roles[1]
62
      DEFAULT_ROLE = roles.last
63
      ROLE_MAPPING = {10 => DEFAULT_ROLE,   # viewer
64
                      25 => DEFAULT_ROLE,   # reporter
65
                      40 => DEFAULT_ROLE,   # updater
66
                      55 => developer_role, # developer
67
                      70 => manager_role,   # manager
68
                      90 => manager_role    # administrator
69
                      }
70

    
71
      CUSTOM_FIELD_TYPE_MAPPING = {0 => 'string', # String
72
                                   1 => 'int',    # Numeric
73
                                   2 => 'int',    # Float
74
                                   3 => 'list',   # Enumeration
75
                                   4 => 'string', # Email
76
                                   5 => 'bool',   # Checkbox
77
                                   6 => 'list',   # List
78
                                   7 => 'list',   # Multiselection list
79
                                   8 => 'date',   # Date
80
                                   }
81

    
82
      RELATION_TYPE_MAPPING = {1 => IssueRelation::TYPE_RELATES,    # related to
83
                               2 => IssueRelation::TYPE_RELATES,    # parent of
84
                               3 => IssueRelation::TYPE_RELATES,    # child of
85
                               0 => IssueRelation::TYPE_DUPLICATES, # duplicate of
86
                               4 => IssueRelation::TYPE_DUPLICATES  # has duplicate
87
                               }
88

    
89
    class MantisUser < ActiveRecord::Base
90
      self.table_name = :mantis_user_table
91

    
92
      def firstname
93
        @firstname = realname.blank? ? username : realname.split.first[0..29]
94
        @firstname
95
      end
96

    
97
      def lastname
98
        @lastname = realname.blank? ? '-' : realname.split[1..-1].join(' ')[0..29]
99
        @lastname = '-' if @lastname.blank?
100
        @lastname
101
      end
102

    
103
      def email
104
        if read_attribute(:email).match(/^([^@\s]+)@((?:[-a-z0-9]+\.)+[a-z]{2,})$/i) &&
105
             !User.find_by_mail(read_attribute(:email))
106
          @email = read_attribute(:email)
107
        else
108
          @email = "#{username}@foo.bar"
109
        end
110
      end
111

    
112
      def username
113
        read_attribute(:username)[0..29].gsub(/[^a-zA-Z0-9_\-@\.]/, '-')
114
      end
115
    end
116

    
117
    class MantisProject < ActiveRecord::Base
118
      self.table_name = :mantis_project_table
119
      has_many :versions, :class_name => "MantisVersion", :foreign_key => :project_id
120
      has_many :categories, :class_name => "MantisCategory", :foreign_key => :project_id
121
      has_many :news, :class_name => "MantisNews", :foreign_key => :project_id
122
      has_many :members, :class_name => "MantisProjectUser", :foreign_key => :project_id
123

    
124
      def identifier
125
        read_attribute(:name).downcase.gsub(/[^a-z0-9\-]+/, '-').slice(0, Project::IDENTIFIER_MAX_LENGTH)
126
      end
127
    end
128

    
129
    class MantisVersion < ActiveRecord::Base
130
      self.table_name = :mantis_project_version_table
131

    
132
      def version
133
        read_attribute(:version)[0..29]
134
      end
135

    
136
      def description
137
        read_attribute(:description)[0..254]
138
      end
139
    end
140

    
141
    class MantisCategory < ActiveRecord::Base
142
      self.table_name = :mantis_category_table
143
    end
144

    
145
    class MantisProjectUser < ActiveRecord::Base
146
      self.table_name = :mantis_project_user_list_table
147
    end
148

    
149
    class MantisBug < ActiveRecord::Base
150
      self.table_name = :mantis_bug_table
151
      belongs_to :bug_text, :class_name => "MantisBugText", :foreign_key => :bug_text_id
152
      has_many :bug_notes, :class_name => "MantisBugNote", :foreign_key => :bug_id
153
      has_many :bug_files, :class_name => "MantisBugFile", :foreign_key => :bug_id
154
      has_many :bug_monitors, :class_name => "MantisBugMonitor", :foreign_key => :bug_id
155
    end
156

    
157
    class MantisBugText < ActiveRecord::Base
158
      self.table_name = :mantis_bug_text_table
159

    
160
      # Adds Mantis steps_to_reproduce and additional_information fields
161
      # to description if any
162
      def full_description
163
        full_description = description
164
        full_description += "\n\n*Steps to reproduce:*\n\n#{steps_to_reproduce}" unless steps_to_reproduce.blank?
165
        full_description += "\n\n*Additional information:*\n\n#{additional_information}" unless additional_information.blank?
166
        full_description
167
      end
168
    end
169

    
170
    class MantisBugNote < ActiveRecord::Base
171
      self.table_name = :mantis_bugnote_table
172
      belongs_to :bug, :class_name => "MantisBug", :foreign_key => :bug_id
173
      belongs_to :bug_note_text, :class_name => "MantisBugNoteText", :foreign_key => :bugnote_text_id
174
    end
175

    
176
    class MantisBugNoteText < ActiveRecord::Base
177
      self.table_name = :mantis_bugnote_text_table
178
    end
179

    
180
    class MantisBugFile < ActiveRecord::Base
181
      self.table_name = :mantis_bug_file_table
182

    
183
      def size
184
        filesize
185
      end
186

    
187
      def original_filename
188
        MantisMigrate.encode(filename)
189
      end
190

    
191
      def content_type
192
        file_type
193
      end
194

    
195
      def read(*args)
196
          if @read_finished
197
              nil
198
          else
199
              @read_finished = true
200
              content
201
          end
202
      end
203
    end
204

    
205
    class MantisBugRelationship < ActiveRecord::Base
206
      self.table_name = :mantis_bug_relationship_table
207
    end
208

    
209
    class MantisBugMonitor < ActiveRecord::Base
210
      self.table_name = :mantis_bug_monitor_table
211
    end
212

    
213
    class MantisNews < ActiveRecord::Base
214
      self.table_name = :mantis_news_table
215
    end
216

    
217
    class MantisCustomField < ActiveRecord::Base
218
      self.table_name = :mantis_custom_field_table
219
      self.inheritance_column = :none
220
      has_many :values, :class_name => "MantisCustomFieldString", :foreign_key => :field_id
221
      has_many :projects, :class_name => "MantisCustomFieldProject", :foreign_key => :field_id
222

    
223
      def format
224
        read_attribute :type
225
      end
226

    
227
      def name
228
        read_attribute(:name)[0..29]
229
      end
230
    end
231

    
232
    class MantisCustomFieldProject < ActiveRecord::Base
233
      self.table_name = :mantis_custom_field_project_table
234
    end
235

    
236
    class MantisCustomFieldString < ActiveRecord::Base
237
      self.table_name = :mantis_custom_field_string_table
238
    end
239

    
240
    def self.migrate
241

    
242
      # Users
243
      print "Migrating users" + "\n"
244
      User.where("login <> 'admin'").delete_all
245
      users_map = {}
246
      users_migrated = 0
247
      MantisUser.all.each do |user|
248
        u = User.new :firstname => encode(user.firstname),
249
                     :lastname => encode(user.lastname),
250
                     :mail => user.email,
251
                     :last_login_on => Time.at(user.last_visit).to_datetime, #KH
252
                     :created_on => Time.at(user.date_created).to_datetime, #KH
253
                     :updated_on => Time.at(user.date_created).to_datetime #KH
254
        u.last_login_on = Time.at(user.last_visit).to_datetime #KH        
255
        u.updated_on =   Time.at(user.date_created).to_datetime #KH
256
        u.created_on =   Time.at(user.date_created).to_datetime #KH   
257
        print "user created: " + encode(Time.at(user.date_created).to_datetime) + "\n"
258
        u.login = user.username
259
        u.password = 'mantis'
260
        u.status = User::STATUS_LOCKED if user.enabled != 1
261
        u.admin = true if user.access_level == 90
262
        next unless u.save!
263
        users_migrated += 1
264
        users_map[user.id] = u.id
265
        print '.'
266
      end
267
      puts
268

    
269
      categories_map = {}
270
      # KH
271
      # Categories
272
      # added KH prior assignment in Migrating projects
273
      # the categories have to exists in categories_id_map to be able to map them 
274
      categories_id_map = {} #KH
275
      print "Migrating Categories" + "\n"
276
      IssueCategory.destroy_all
277
      MantisCategory.all.each do |category|
278
          print 'category: ' + encode(category.name) + "\n" # KH
279
          g = IssueCategory.new :name => category.name, # KH was category.category[0,30]  
280
                                :project_id => category.project_id
281
          g.save          
282
          categories_id_map[category.id] = category.name # KH 
283
          categories_map[category.name] = g.id
284
      end
285
      # _KH
286
      
287
      # Projects
288
      print "Migrating projects" + "\n"
289
      Project.destroy_all
290
      projects_map = {}
291
      versions_map = {}
292
      
293
      
294
      MantisProject.all.each do |project|
295
        print encode(project.name)+"\n" # KH
296
        p = Project.new :name => encode(project.name),
297
                        :description => encode(project.description)
298
        p.identifier = project.identifier
299
        next unless p.save
300
        projects_map[project.id] = p.id
301
        p.enabled_module_names = ['issue_tracking', 'news', 'wiki']
302
        p.trackers << TRACKER_BUG unless p.trackers.include?(TRACKER_BUG)
303
        p.trackers << TRACKER_FEATURE unless p.trackers.include?(TRACKER_FEATURE)
304
        print '.'
305

    
306
        # Project members
307
        project.members.each do |member|
308
          print 'User ID: ' + encode(member.user_id) +"\n" # KH
309
          m = Member.new :user => User.find_by_id(users_map[member.user_id]),
310
                           :roles => [ROLE_MAPPING[member.access_level] || DEFAULT_ROLE]
311
          m.project = p
312
          m.save
313
        end
314

    
315
        # Project versions
316
        project.versions.each do |version|
317
          v = Version.new :name => encode(version.version),
318
                          :description => encode(version.description),
319
                          :effective_date => (version.date_order ? version.date_order.to_date : nil)
320
          v.project = p
321
          v.save
322
          versions_map[version.id] = v.id
323
        end
324

    
325
        #Project categories
326
        
327
        #project.categories.each do |category|
328
        #  print 'category: ' + encode(category.name) + "\n" # KH
329
        #  g = IssueCategory.new :name => category.name[0,30] # KH was category.category[0,30]
330
        #                          :project_id => category.
331
        #  g.project = p
332
        #  g.save
333
        #  categories_map[category.name] = g.id # KH was category.category
334
          
335
        #end                
336
      end
337
      puts
338

    
339
      # Bugs
340
      print "Migrating bugs"
341
      Issue.destroy_all
342
      issues_map = {}
343
      keep_bug_ids = (Issue.count == 0)
344
      MantisBug.find_each(:batch_size => 200) do |bug|
345
        print "Bug: " + encode(bug.summary) + "\n"#KH        
346
        next unless projects_map[bug.project_id] && users_map[bug.reporter_id]
347
        
348
        i = Issue.new :project_id => projects_map[bug.project_id],
349
                      :subject => encode(bug.summary),
350
                      :description => encode(bug.bug_text.full_description),
351
                      :priority => PRIORITY_MAPPING[bug.priority] || DEFAULT_PRIORITY,
352
                      :created_on => Time.at(bug.date_submitted).to_datetime, # KH
353
                      :updated_on => Time.at(bug.last_updated).to_datetime, #KH
354
                       :category_id => bug.category_id
355
        i.created_on = Time.at(bug.date_submitted.dup).to_datetime # KH       
356
        i.updated_on = Time.at(bug.date_submitted.dup).to_datetime # KH
357
        i.author = User.find_by_id(users_map[bug.reporter_id])
358
        print "Author: " + encode(i.author) + "\n" #KH        
359
        print "Category Name" + encode(categories_id_map[bug.category_id]) + "\n"
360
        #i.category = IssueCategory.find_by_project_id_and_name(i.project_id, bug.category[0,30]) unless bug.category.blank?
361
        #i.category_id = IssueCategory.find_by_project_id_and_name(i.project_id, categories_id_map[bug.category_id][0,30]) unless bug.category_id.blank? # KH
362
        i.category_id = categories_map[categories_id_map[bug.category_id]]
363
        print "i.category: " + encode(i.category_id) + "\n" #KH 
364
        i.fixed_version = Version.find_by_project_id_and_name(i.project_id, bug.fixed_in_version) unless bug.fixed_in_version.blank?
365
        i.tracker = (bug.severity == 10 ? TRACKER_FEATURE : TRACKER_BUG) # KH was typo: bug.severfity
366
        i.status = STATUS_MAPPING[bug.status] || i.status
367
        i.id = bug.id if keep_bug_ids
368
        issues_map[bug.id] = i.id
369
        next unless i.save        
370
        print '.'
371
        STDOUT.flush
372

    
373
        # Assignee
374
        # Redmine checks that the assignee is a project member
375
        if (bug.handler_id && users_map[bug.handler_id])
376
          i.assigned_to = User.find_by_id(users_map[bug.handler_id])
377
          i.save(:validate => false)
378
        end
379
 
380
        # Bug notes
381
        bug.bug_notes.each do |note|
382
          print "Bug Notes: " + "\n" #KH 
383
          next unless users_map[note.reporter_id]
384
          n = Journal.new :notes => encode(note.bug_note_text.note),
385
                          :created_on => Time.at(note.date_submitted).to_datetime #KH
386
          #print "Bug Notes created on: " + encode(n.created_on.dup)+ "\n" #KH 
387
          #n.created_on = Time.at(note.date_submitted.dup).to_datetime #KH
388
          #print "Bug Notes created on: " + encode(n.created_on.dup)+ "\n" #KH 
389
          n.user = User.find_by_id(users_map[note.reporter_id])
390
          n.journalized = i
391
          n.save
392
        end
393

    
394
        # Bug files
395
        bug.bug_files.each do |file|
396
          a = Attachment.new :created_on => Time.at(file.date_added).to_datetime #KH
397
          a.created_on = Time.at(file.date_added).to_datetime #KH
398
          a.file = file
399
          a.author = User.first
400
          a.container = i
401
          a.save
402
        end
403

    
404
        # Bug monitors
405
        bug.bug_monitors.each do |monitor|
406
          next unless users_map[monitor.user_id]
407
          i.add_watcher(User.find_by_id(users_map[monitor.user_id]))
408
        end
409
      end
410

    
411
      # update issue id sequence if needed (postgresql)
412
      Issue.connection.reset_pk_sequence!(Issue.table_name) if Issue.connection.respond_to?('reset_pk_sequence!')
413
      puts
414

    
415
      # Bug relationships
416
      print "Migrating bug relations"
417
      MantisBugRelationship.all.each do |relation|
418
        next unless issues_map[relation.source_bug_id] && issues_map[relation.destination_bug_id]
419
        r = IssueRelation.new :relation_type => RELATION_TYPE_MAPPING[relation.relationship_type]
420
        r.issue_from = Issue.find_by_id(issues_map[relation.source_bug_id])
421
        r.issue_to = Issue.find_by_id(issues_map[relation.destination_bug_id])
422
        pp r unless r.save
423
        print '.'
424
        STDOUT.flush
425
      end
426
      puts
427

    
428
      # News
429
      print "Migrating news"
430
      News.destroy_all
431
      MantisNews.where('project_id > 0').all.each do |news|
432
        next unless projects_map[news.project_id]
433
        n = News.new :project_id => projects_map[news.project_id],
434
                     :title => encode(news.headline[0..59]),
435
                     :description => encode(news.body),
436
                     :created_on => news.date_posted
437
        n.author = User.find_by_id(users_map[news.poster_id])
438
        n.save
439
        print '.'
440
        STDOUT.flush
441
      end
442
      puts
443

    
444
      # Custom fields
445
      print "Migrating custom fields"
446
      IssueCustomField.destroy_all
447
      MantisCustomField.all.each do |field|
448
        f = IssueCustomField.new :name => field.name[0..29],
449
                                 :field_format => CUSTOM_FIELD_TYPE_MAPPING[field.format],
450
                                 :min_length => field.length_min,
451
                                 :max_length => field.length_max,
452
                                 :regexp => field.valid_regexp,
453
                                 :possible_values => field.possible_values.split('|'),
454
                                 :is_required => field.require_report?
455
        next unless f.save
456
        print '.'
457
        STDOUT.flush
458
        # Trackers association
459
        f.trackers = Tracker.all
460

    
461
        # Projects association
462
        field.projects.each do |project|
463
          f.projects << Project.find_by_id(projects_map[project.project_id]) if projects_map[project.project_id]
464
        end
465

    
466
        # Values
467
        field.values.each do |value|
468
          v = CustomValue.new :custom_field_id => f.id,
469
                              :value => value.value
470
          v.customized = Issue.find_by_id(issues_map[value.bug_id]) if issues_map[value.bug_id]
471
          v.save
472
        end unless f.new_record?
473
      end
474
      puts
475

    
476
      puts
477
      puts "Users:           #{users_migrated}/#{MantisUser.count}"
478
      puts "Projects:        #{Project.count}/#{MantisProject.count}"
479
      puts "Memberships:     #{Member.count}/#{MantisProjectUser.count}"
480
      puts "Versions:        #{Version.count}/#{MantisVersion.count}"
481
      puts "Categories:      #{IssueCategory.count}/#{MantisCategory.count}"
482
      puts "Bugs:            #{Issue.count}/#{MantisBug.count}"
483
      puts "Bug notes:       #{Journal.count}/#{MantisBugNote.count}"
484
      puts "Bug files:       #{Attachment.count}/#{MantisBugFile.count}"
485
      puts "Bug relations:   #{IssueRelation.count}/#{MantisBugRelationship.count}"
486
      puts "Bug monitors:    #{Watcher.count}/#{MantisBugMonitor.count}"
487
      puts "News:            #{News.count}/#{MantisNews.count}"
488
      puts "Custom fields:   #{IssueCustomField.count}/#{MantisCustomField.count}"
489
    end
490

    
491
    def self.encoding(charset)
492
      @charset = charset
493
    end
494

    
495
    def self.establish_connection(params)
496
      constants.each do |const|
497
        klass = const_get(const)
498
        next unless klass.respond_to? 'establish_connection'
499
        klass.establish_connection params
500
      end
501
    end
502

    
503
    def self.encode(text)
504
      text.to_s.force_encoding(@charset).encode('UTF-8')
505
    end
506
  end
507

    
508
  puts
509
  if Redmine::DefaultData::Loader.no_data?
510
    puts "Redmine configuration need to be loaded before importing data."
511
    puts "Please, run this first:"
512
    puts
513
    puts "  rake redmine:load_default_data RAILS_ENV=\"#{ENV['RAILS_ENV']}\""
514
    exit
515
  end
516

    
517
  puts "WARNING: Your Redmine data will be deleted during this process."
518
  print "Are you sure you want to continue ? [y/N] "
519
  STDOUT.flush
520
  break unless STDIN.gets.match(/^y$/i)
521

    
522
  # Default Mantis database settings
523
  db_params = {:adapter => 'mysql2',
524
               :database => 'bugtracker',
525
               :host => 'localhost',
526
               :username => 'root',
527
               :password => '' }
528

    
529
  puts
530
  puts "Please enter settings for your Mantis database"
531
  [:adapter, :host, :database, :username, :password].each do |param|
532
    print "#{param} [#{db_params[param]}]: "
533
    value = STDIN.gets.chomp!
534
    db_params[param] = value unless value.blank?
535
  end
536

    
537
  while true
538
    print "encoding [UTF-8]: "
539
    STDOUT.flush
540
    encoding = STDIN.gets.chomp!
541
    encoding = 'UTF-8' if encoding.blank?
542
    break if MantisMigrate.encoding encoding
543
    puts "Invalid encoding!"
544
  end
545
  puts
546

    
547
  # Make sure bugs can refer bugs in other projects
548
  Setting.cross_project_issue_relations = 1 if Setting.respond_to? 'cross_project_issue_relations'
549

    
550
  old_notified_events = Setting.notified_events
551
  old_password_min_length = Setting.password_min_length
552
  begin
553
    # Turn off email notifications temporarily
554
    Setting.notified_events = []
555
    Setting.password_min_length = 4
556
    # Run the migration
557
    MantisMigrate.establish_connection db_params
558
    MantisMigrate.migrate
559
  ensure
560
    # Restore previous settings
561
    Setting.notified_events = old_notified_events
562
    Setting.password_min_length = old_password_min_length
563
  end
564

    
565
end
566
end