0002-implements-background-issue-PDF-and-CSV-exports.patch

Jens Krämer, 2019-03-21 11:27

Download (21.4 KB)

View differences:

app/controllers/issues_controller.rb
65 65
          render_feed(@issues, :title => "#{@project || Setting.app_title}: #{l(:label_issue_plural)}")
66 66
        }
67 67
        format.csv  {
68
          @issues = @query.issues(:limit => Setting.issues_export_limit.to_i)
69
          send_data(query_to_csv(@issues, @query, params[:csv]), :type => 'text/csv; header=present', :filename => 'issues.csv')
68
          handle_issues_export
69
          return
70 70
        }
71 71
        format.pdf  {
72
          @issues = @query.issues(:limit => Setting.issues_export_limit.to_i)
73
          send_file_headers! :type => 'application/pdf', :filename => 'issues.pdf'
72
          handle_issues_export
73
          return
74 74
        }
75 75
      end
76 76
    else
......
603 603
      redirect_back_or_default issue_path(@issue)
604 604
    end
605 605
  end
606

  
607
  # exports with less than this number of issues are done inline (not handed
608
  # over to the configured AJ backend at all)
609
  PERFORM_NOW_LIMIT = 1000
610

  
611
  # Issues CSV / PDF export via ActiveJob
612
  # See Redmine::Exports::BackgroundExport for how this works.
613
  def handle_issues_export
614
    limit = Setting.issues_export_limit.to_i
615
    issue_count = @query.issue_count
616
    export = Redmine::Export::BackgroundExport.new(
617
      @query,
618
      worker_class: Redmine::Export::IssueExport,
619
      perform_now: issue_count <= PERFORM_NOW_LIMIT,
620
      params: {
621
        format: params[:format],
622
        options: {
623
          project_id: @project.try(:identifier),
624
          limit: limit,
625
          csv: { encoding: params[:encoding] }.merge(params[:csv]||{})
626
        }
627
      }
628
    )
629

  
630
    if export.wait
631
      if attachment = export.result
632
        send_file attachment.diskfile,
633
                  filename: Redmine::Export::IssueExport.filename(params[:format]),
634
                  type: attachment.content_type,
635
                  disposition: 'attachment'
636
        return
637
      else
638
        flash[:error] = l(:error_issue_export_failed)
639
      end
640
    else
641
      flash[:notice] = l(:notice_issue_export_scheduled)
642
    end
643

  
644
    params[:format] = nil
645
    parameters = params.permit!.to_h.except(:action, :controller, :csv, :project_id, :format, :encoding)
646

  
647
    if @project
648
      redirect_to project_issues_url(@project, parameters)
649
    else
650
      redirect_to issues_url(parameters)
651
    end
652
  end
653

  
654

  
606 655
end
app/jobs/export_job.rb
1
# frozen_string_literal: true
2

  
3
# Background job for creating CSV/PDF exports.
4
#
5
class ExportJob < ActiveJob::Base
6
  include Redmine::I18n
7

  
8
  def perform(filename, query_hash, user_id, created_at,
9
              worker_class_name, params, notify_if_finished_after = nil)
10

  
11
    User.current = User.anonymous
12
    if user = User.active.find_by_id(user_id)
13
      User.current = user
14
      set_language_if_valid user.language || Setting.default_language
15
    end
16

  
17
    @notify_if_finished_after = notify_if_finished_after.to_i
18

  
19
    worker_class = worker_class_name.constantize
20
    export = worker_class.new(query_hash, params)
21

  
22

  
23
    if data = export.generate_data
24
      file = DataFile.new data, filename, export.content_type
25
      @attachment = Attachment.new(file: file, author: User.current)
26
      @attachment.skip_filesize_validation!
27
      unless @attachment.save
28
        @error = @attachment.errors.full_messages.join("\n").presence
29
      end
30
    else
31
      @error = I18n.t(:error_export_failed)
32
    end
33

  
34
    if notify?
35
      if @attachment && @error.blank?
36
        # the controller has given up waiting now, so there is no need for the
37
        # unique filename anymore. Let's change it to something nice. This
38
        # allows us to have URLs like attachments/download/100/issues.pdf
39
        # instead of attachments/download/100/<uuid>.pdf
40
        @attachment.update_column :filename, export.filename
41
      end
42
      Mailer.export_notification(
43
        User.current, Time.at(created_at), @attachment, @error
44
      ).deliver
45
    end
46

  
47
  end
48

  
49

  
50
  private
51

  
52
  def notify?
53
    if @notify_if_finished_after > 0
54
      # 1 second margin to avoid lost exports. This may lead to
55
      # a notification being sent out even if the file was delivered
56
      # directly but this is the lesser evil.
57
      Time.at(@notify_if_finished_after) - 1.second < Time.now
58
    else
59
      false
60
    end
61
  end
62

  
63
  # StringIO plus filename and content_type.
64
  # The result of #generate_data should be of this kind
65
  class DataFile < StringIO
66
    attr_reader :original_filename, :content_type
67
    def initialize(string, filename, content_type)
68
      super string
69
      @original_filename = filename
70
      @content_type = content_type
71
    end
72
  end
73

  
74

  
75
end
app/models/attachment.rb
70 70
  end
71 71

  
72 72
  def validate_max_file_size
73
    return if @skip_filesize_validation
73 74
    if @temp_file && self.filesize > Setting.attachment_max_size.to_i.kilobytes
74 75
      errors.add(:base, l(:error_attachment_too_big, :max_size => Setting.attachment_max_size.to_i.kilobytes))
75 76
    end
76 77
  end
77 78

  
79
  def skip_filesize_validation!
80
    @skip_filesize_validation = true
81
  end
82

  
78 83
  def validate_file_extension
79 84
    if @temp_file
80 85
      extension = File.extname(filename)
app/models/mailer.rb
602 602
    end
603 603
  end
604 604

  
605
  def export_notification(user, date, attachment, error)
606
    set_language_if_valid(user.language)
607
    @date = date
608
    @error = error
609
    if attachment
610
      @filename = attachment.filename
611
      @url = download_named_attachment_url(attachment, @filename)
612
    else
613
      @url = issues_url
614
    end
615
    redmine_headers 'Sender' => user.login, 'Url' => @url
616

  
617
    mail to: user,
618
         subject: l(error.blank? ?
619
                    :mail_subject_export_finished :
620
                    :mail_subject_export_failed)
621
  end
622

  
605 623
  # Activates/desactivates email deliveries during +block+
606 624
  def self.with_deliveries(enabled = true, &block)
607 625
    was_enabled = ActionMailer::Base.perform_deliveries
app/views/mailer/export_notification.html.erb
1
<% if @error.present? %>
2
  <p><%= l :mail_body_export_failed, date: format_time(@date) %></p>
3
  <p><%= @error %></p>
4
<% else %>
5
  <p><%= l :mail_body_export_finished, date: format_time(@date) %></p>
6
  <p><%= link_to @filename, @url %></p>
7
<% end %>
8

  
app/views/mailer/export_notification.text.erb
1
<% if @error.present? %>
2
<%= l :mail_body_export_failed, date: format_time(@date) %>
3
<%= @error %>
4
<% else %>
5
<%= l :mail_body_export_finished, date: format_time(@date) %>
6

  
7
<%= @url %>
8
<% end %>
config/locales/en.yml
187 187
  notice_new_password_must_be_different: The new password must be different from the current password
188 188
  notice_import_finished: "%{count} items have been imported"
189 189
  notice_import_finished_with_errors: "%{count} out of %{total} items could not be imported"
190
  notice_issue_export_scheduled: "Exporting your issues takes longer than usual andwill be continued in the background. You will be notified once the process is completed."
190 191

  
191 192
  error_can_t_load_default_data: "Default configuration could not be loaded: %{value}"
192 193
  error_scm_not_found: "The entry or revision was not found in the repository."
......
225 226
  error_can_not_delete_auth_source: "This authentication mode is in use and cannot be deleted."
226 227
  error_spent_on_future_date: "Cannot log time on a future date"
227 228
  error_not_allowed_to_log_time_for_other_users: "You are not allowed to log time for other users"
229
  error_issue_export_failed: The issue export failed.
228 230

  
229 231
  mail_subject_lost_password: "Your %{value} password"
230 232
  mail_body_lost_password: 'To change your password, click on the following link:'
......
250 252
  mail_body_security_notification_notify_disabled: "Email address %{value} no longer receives notifications."
251 253
  mail_body_settings_updated: "The following settings were changed:"
252 254
  mail_body_password_updated: "Your password has been changed."
255
  mail_subject_export_failed: Export failed
256
  mail_body_export_failed: The export that was started by you on %{date} has failed.
257
  mail_subject_export_finished: Export finished
258
  mail_body_export_finished: "The export that was started by you on %{date} is available for download now:"
253 259

  
254 260
  field_name: Name
255 261
  field_description: Description
lib/redmine/export/background_export.rb
1
# frozen_string_literal: true
2

  
3
require 'timeout'
4
require 'securerandom'
5

  
6
module Redmine
7
  module Export
8

  
9
    # This class models an export that may or may not take too long to wait for
10
    # in a web request.
11
    #
12
    # The initializer creates an export that will result in an attachment with
13
    # a random filename. Use the wait instance methods to wait at most the
14
    # specified number of seconds for completion of the job. If this returns
15
    # true, the generated file is available as an Attachment object through
16
    # #result. Otherwise, the export is taking longer and the user will be
17
    # notified by email.
18
    #
19
    # This makes most sense with a background job runner like DelayedJob
20
    # configured for ActiveJob. Without this, the job will be run immediately
21
    # and everything will be like before - #initialize (which launches the
22
    # background job) will block until the export is done, and wait will return
23
    # immediately true since the attachments exists when calling code gets
24
    # around to call it.
25
    #
26
    # Jobs can be forced to run inline via the perform_now: argument. This is
27
    # used in IssuesController to always run small exports immediately instead
28
    # of putting them in a queue where they potentially have to wait
29
    # unnecessarily long for large exports to complete.
30
    class BackgroundExport
31

  
32
      def initialize(query, wait_for: 30.seconds,
33
                     user: User.current,
34
                     perform_now: false,
35
                     worker_class: Redmine::Export::IssueExport,
36
                     params: {})
37

  
38
        @wait_for = wait_for
39
        @filename = SecureRandom.uuid
40
        @user = user
41
        ExportJob.send(
42
          perform_now ? :perform_now : :perform_later,
43
          @filename,
44
          serialize_query(query), user.id, Time.now.to_i,
45
          worker_class.name, params,
46
          wait_for.from_now.to_i
47
        )
48
      end
49

  
50
      def serialize_query(query)
51
        query.as_params.tap do |params|
52
          params[:c].map!(&:to_s) if params[:c]
53
        end
54
      end
55

  
56
      # returns true if the export finished in time
57
      def wait
58
        Timeout.timeout(@wait_for) {
59
          sleep 1 while not finished?
60
          true
61
        }
62
      rescue Timeout::Error
63
        false
64
      end
65

  
66
      def result
67
        scope.first
68
      end
69

  
70
      private
71

  
72
      def finished?
73
        Attachment.uncached { scope.any? }
74
      end
75

  
76
      def scope
77
        Attachment.where(author: @user, filename: @filename)
78
      end
79

  
80
    end
81
  end
82
end
lib/redmine/export/issue_export.rb
1
# frozen_string_literal: true
2

  
3
module Redmine
4
  module Export
5
    class IssueExport
6
      include IssuesHelper
7
      include QueriesHelper
8
      include Redmine::Export::PDF::IssuesPdfHelper
9

  
10
      def initialize(query_hash, params)
11
        @format = params[:format]
12
        unless %w(csv pdf).include? @format
13
          raise "Cannot handle export format: #{params[:format]}"
14
        end
15

  
16
        if options = params[:options]
17
          @project_id    = options[:project_id]
18
          @limit         = options[:limit]
19
          @csv_options   = options[:csv]
20
        end
21

  
22
        @project = Project.find @project_id if @project_id
23

  
24
        if id = query_hash[:query_id]
25
          @query = IssueQuery.find id
26
        else
27
          @query = IssueQuery.new name: "_"
28
          @query.build_from_params(query_hash)
29
        end
30
        @query.project = @project
31
      end
32

  
33
      # expected by QueriesHelper#query_to_csv
34
      def params
35
        @csv_options || {}
36
      end
37

  
38
      def generate_data
39
        issues = @query.lazy_issues limit: @limit
40

  
41
        case @format
42
        when "csv"
43
          query_to_csv(issues, @query, @csv_options)
44
        when "pdf"
45
          issues_to_pdf(issues, @project, @query)
46
        end
47
      end
48

  
49
      def content_type
50
        self.class.content_type @format
51
      end
52

  
53
      def filename
54
        self.class.filename @format
55
      end
56

  
57

  
58
      # format potentially is params[:format] which may be something
59
      # unexpected, which is why we dont just use "issues#{format}".
60
      def self.filename(format)
61
        case format
62
        when "csv"
63
          "issues.csv"
64
        when "pdf"
65
          "issues.pdf"
66
        end
67
      end
68

  
69
      def self.content_type(format)
70
        case format
71
        when "csv"
72
          "text/csv; header=present"
73
        when "pdf"
74
          "application/pdf"
75
        end
76
      end
77

  
78

  
79
    end
80
  end
81
end
test/unit/export_job_test.rb
1
require_relative '../test_helper'
2

  
3
class DummyWorker
4
  def initialize(hsh, params)
5
    @data = hsh[:data] + params[:more_data]
6
  end
7

  
8
  def generate_data
9
    @data
10
  end
11
  def content_type
12
    'text/plain'
13
  end
14
  def filename
15
    'dummy.txt'
16
  end
17
end
18

  
19
class ExportJobTest < ActiveJob::TestCase
20
  fixtures :users
21

  
22
  setup do
23
    set_tmp_attachments_directory
24
  end
25

  
26
  test 'should call worker class and generate attachment' do
27
    assert_difference 'Attachment.count' do
28
      ExportJob.perform_now(
29
        'random-filename', {data: 'some test'}, 1, Time.now.to_i,
30
        'DummyWorker', {more_data: ' data'}
31
      )
32
    end
33
    assert a = Attachment.find_by_filename('random-filename')
34
    assert_equal 1, a.author_id
35
    assert_equal 'text/plain', a.content_type
36
    assert_equal "some test data", IO.read(a.diskfile)
37
  end
38
end
test/unit/lib/redmine/export/background_export_test.rb
1
require_relative '../../../../test_helper'
2

  
3
class BackgroundExportTest < ActiveJob::TestCase
4
  fixtures :projects, :enabled_modules, :users, :members,
5
           :member_roles, :roles, :trackers, :issue_statuses,
6
           :issue_categories, :enumerations, :issues,
7
           :watchers, :custom_fields, :custom_values, :versions,
8
           :queries,
9
           :projects_trackers,
10
           :custom_fields_trackers,
11
           :workflows
12

  
13
  setup do
14
    @query = IssueQuery.new name: '_'
15
    @project = Project.find 1
16
    User.current = @user = User.find 1
17
  end
18

  
19
  test 'should enqueue export job' do
20
    assert_enqueued_with(job: ExportJob) do
21
      Redmine::Export::BackgroundExport.new @query,
22
                           params: {format: 'csv', options:{limit: 10000}}
23
    end
24
  end
25

  
26
  test 'wait should wait and return false if not finished' do
27
    e = Redmine::Export::BackgroundExport.new @query,
28
                             wait_for: 1.second,
29
                             params: {format: 'csv', options:{limit: 10000}}
30
    t = Time.now
31
    refute e.wait
32
    assert Time.now - t > 1.second
33
  end
34

  
35
  test 'wait should return true if finished' do
36
    e = Redmine::Export::BackgroundExport.new @query,
37
                             params: {format: 'csv', options:{limit: 10000}}
38
    class << e
39
      def finished?; true end
40
    end
41

  
42
    Timeout.timeout(1) { assert e.wait }
43
  end
44

  
45
end
test/unit/lib/redmine/export/issue_export_test.rb
1
require_relative '../../../../test_helper'
2

  
3
class IssueExportTest < ActiveSupport::TestCase
4
  fixtures :projects, :enabled_modules, :users, :members,
5
           :member_roles, :roles, :trackers, :issue_statuses,
6
           :issue_categories, :enumerations, :issues,
7
           :watchers, :custom_fields, :custom_values, :versions,
8
           :queries,
9
           :projects_trackers,
10
           :custom_fields_trackers,
11
           :workflows
12

  
13
  setup do
14
    @query = IssueQuery.new name: '_'
15
    @project = Project.find 1
16
    User.current = @user = User.find 1
17
  end
18

  
19
  test 'should generate issues pdf' do
20
    export = Redmine::Export::IssueExport.new @query.as_params,
21
                             format: 'pdf',
22
                             options: { limit: 10000 }
23
    assert data = export.generate_data
24
    assert data.length > 0
25
    assert_equal 'issues.pdf', export.filename
26
  end
27

  
28
  test 'should generate issues csv with new query' do
29
    export = Redmine::Export::IssueExport.new @query.as_params,
30
                             format: 'csv',
31
                             options: { limit: 10000 }
32
    assert data = export.generate_data
33
    assert data.length > 0
34
    assert_equal 'issues.csv', export.filename
35

  
36
    assert s = data.lines
37
    s.shift # headers
38
    assert_equal Issue.visible.open.count, s.size
39
  end
40

  
41
  test 'should honor filter and project' do
42
    @query.add_filter("issue_id", '><', ['2','3'])
43

  
44
    export = Redmine::Export::IssueExport.new @query.as_params,
45
                             format: 'csv',
46
                             options: { limit: 10000, project_id: 1 }
47
    assert data = export.generate_data
48
    assert s = data.lines
49
    s.shift # headers
50
    assert_equal (@project.issues.visible.open.count - 2), s.size
51
  end
52

  
53
  test 'should honor limit' do
54
    export = Redmine::Export::IssueExport.new @query.as_params,
55
                             format: 'csv',
56
                             options: { limit: 2 }
57
    assert data = export.generate_data
58
    assert s = data.lines
59
    assert_equal 3, s.size
60
  end
61

  
62
  test 'should honor sorting' do
63
    @query.sort_criteria = [['id', 'desc']]
64
    export = Redmine::Export::IssueExport.new @query.as_params, format: 'csv'
65
    assert data = export.generate_data
66
    assert s = data.lines
67
    s.shift # headers
68
    ids = s.map{|l| l.split(',').first.to_i}
69
    assert_equal ids, ids.sort{|a, b| b <=> a}
70

  
71

  
72
    @query.sort_criteria = [['id', 'asc']]
73
    export = Redmine::Export::IssueExport.new @query.as_params, format: 'csv'
74
    assert data = export.generate_data
75
    assert s = data.lines
76
    s.shift # headers
77
    assert s.many?
78
    ids = s.map{|l| l.split(',').first.to_i}
79
    assert_equal ids, ids.sort{|a, b| a <=> b}
80
  end
81

  
82
end
83

  
0
-