summaryrefslogtreecommitdiffstats
path: root/app/models/import.rb
blob: 28c76268d8275d896463596d98767eecc31822be (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
# frozen_string_literal: true

# Redmine - project management software
# Copyright (C) 2006-2017  Jean-Philippe Lang
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

require 'csv'

class Import < ActiveRecord::Base
  has_many :items, :class_name => 'ImportItem', :dependent => :delete_all
  belongs_to :user
  serialize :settings

  before_destroy :remove_file

  validates_presence_of :filename, :user_id
  validates_length_of :filename, :maximum => 255

  DATE_FORMATS = [
    '%Y-%m-%d',
    '%d/%m/%Y',
    '%m/%d/%Y',
    '%d.%m.%Y',
    '%d-%m-%Y'
  ]

  def initialize(*args)
    super
    self.settings ||= {}
  end

  def file=(arg)
    return unless arg.present? && arg.size > 0

    self.filename = generate_filename
    Redmine::Utils.save_upload(arg, filepath)
  end

  def set_default_settings
    separator = lu(user, :general_csv_separator)
    if file_exists?
      begin
        content = File.read(filepath, 256)
        separator = [',', ';'].sort_by {|sep| content.count(sep) }.last
      rescue Exception => e
      end
    end
    wrapper = '"'
    encoding = lu(user, :general_csv_encoding)

    date_format = lu(user, "date.formats.default", :default => "foo")
    date_format = DATE_FORMATS.first unless DATE_FORMATS.include?(date_format)

    self.settings.merge!(
      'separator' => separator,
      'wrapper' => wrapper,
      'encoding' => encoding,
      'date_format' => date_format
    )
  end

  def to_param
    filename
  end

  # Returns the full path of the file to import
  # It is stored in tmp/imports with a random hex as filename
  def filepath
    if filename.present? && filename =~ /\A[0-9a-f]+\z/
      File.join(Rails.root, "tmp", "imports", filename)
    else
      nil
    end
  end

  # Returns true if the file to import exists
  def file_exists?
    filepath.present? && File.exists?(filepath)
  end

  # Returns the headers as an array that
  # can be used for select options
  def columns_options(default=nil)
    i = -1
    headers.map {|h| [h, i+=1]}
  end

  # Parses the file to import and updates the total number of items
  def parse_file
    count = 0
    read_items {|row, i| count=i}
    update_attribute :total_items, count
    count
  end

  # Reads the items to import and yields the given block for each item
  def read_items
    i = 0
    headers = true
    read_rows do |row|
      if i == 0 && headers
        headers = false
        next
      end
      i+= 1
      yield row, i if block_given?
    end
  end

  # Returns the count first rows of the file (including headers)
  def first_rows(count=4)
    rows = []
    read_rows do |row|
      rows << row
      break if rows.size >= count
    end
    rows
  end

  # Returns an array of headers
  def headers
    first_rows(1).first || []
  end

  # Returns the mapping options
  def mapping
    settings['mapping'] || {}
  end

  # Adds a callback that will be called after the item at given position is imported
  def add_callback(position, name, *args)
    settings['callbacks'] ||= {}
    settings['callbacks'][position.to_i] ||= []
    settings['callbacks'][position.to_i] << [name, args]
    save!
  end

  # Executes the callbacks for the given object
  def do_callbacks(position, object)
    if callbacks = (settings['callbacks'] || {}).delete(position)
      callbacks.each do |name, args|
        send "#{name}_callback", object, *args
      end
      save!
    end
  end

  # Imports items and returns the position of the last processed item
  def run(options={})
    max_items = options[:max_items]
    max_time = options[:max_time]
    current = 0
    imported = 0
    resume_after = items.maximum(:position) || 0
    interrupted = false
    started_on = Time.now

    read_items do |row, position|
      if (max_items && imported >= max_items) || (max_time && Time.now >= started_on + max_time)
        interrupted = true
        break
      end
      if position > resume_after
        item = items.build
        item.position = position

        if object = build_object(row, item)
          if object.save
            item.obj_id = object.id
          else
            item.message = object.errors.full_messages.join("\n")
          end
        end

        item.save!
        imported += 1

        do_callbacks(item.position, object)
      end
      current = position
    end

    if imported == 0 || interrupted == false
      if total_items.nil?
        update_attribute :total_items, current
      end
      update_attribute :finished, true
      remove_file
    end

    current
  end

  def unsaved_items
    items.where(:obj_id => nil)
  end

  def saved_items
    items.where("obj_id IS NOT NULL")
  end

  private

  def read_rows
    return unless file_exists?

    csv_options = {:headers => false}
    csv_options[:encoding] = settings['encoding'].to_s.presence || 'UTF-8'
    csv_options[:encoding] = 'bom|UTF-8' if csv_options[:encoding] == 'UTF-8'
    separator = settings['separator'].to_s
    csv_options[:col_sep] = separator if separator.size == 1
    wrapper = settings['wrapper'].to_s
    csv_options[:quote_char] = wrapper if wrapper.size == 1

    CSV.foreach(filepath, csv_options) do |row|
      yield row if block_given?
    end
  end

  def row_value(row, key)
    if index = mapping[key].presence
      row[index.to_i].presence
    end
  end

  def row_date(row, key)
    if s = row_value(row, key)
      format = settings['date_format']
      format = DATE_FORMATS.first unless DATE_FORMATS.include?(format)
      Date.strptime(s, format) rescue s
    end
  end

  # Builds a record for the given row and returns it
  # To be implemented by subclasses
  def build_object(row)
  end

  # Generates a filename used to store the import file
  def generate_filename
    Redmine::Utils.random_hex(16)
  end

  # Deletes the import file
  def remove_file
    if file_exists?
      begin
        File.delete filepath
      rescue Exception => e
        logger.error "Unable to delete file #{filepath}: #{e.message}" if logger
      end
    end
  end

  # Returns true if value is a string that represents a true value
  def yes?(value)
    value == lu(user, :general_text_yes) || value == '1'
  end
end