##// END OF EJS Templates
jro_apps » redmine
RSS
Merged r15357 (#22583)....
Jean-Philippe Lang -
r14976:0123c1dcb42d
parent child
Show More
Side by Side Unified
Context file:
r14976:0123c1dcb42d Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,249 +1,249
1 # Redmine - project management software
1 # Redmine - project management software
2 # Copyright (C) 2006-2016 Jean-Philippe Lang
2 # Copyright (C) 2006-2016 Jean-Philippe Lang
3 #
3 #
4 # This program is free software; you can redistribute it and/or
4 # This program is free software; you can redistribute it and/or
5 # modify it under the terms of the GNU General Public License
5 # modify it under the terms of the GNU General Public License
6 # as published by the Free Software Foundation; either version 2
6 # as published by the Free Software Foundation; either version 2
7 # of the License, or (at your option) any later version.
7 # of the License, or (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software
15 # along with this program; if not, write to the Free Software
16 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17
17
18 require 'csv'
18 require 'csv'
19
19
20 class Import < ActiveRecord::Base
20 class Import < ActiveRecord::Base
21 has_many :items, :class_name => 'ImportItem', :dependent => :delete_all
21 has_many :items, :class_name => 'ImportItem', :dependent => :delete_all
22 belongs_to :user
22 belongs_to :user
23 serialize :settings
23 serialize :settings
24
24
25 before_destroy :remove_file
25 before_destroy :remove_file
26
26
27 validates_presence_of :filename, :user_id
27 validates_presence_of :filename, :user_id
28 validates_length_of :filename, :maximum => 255
28 validates_length_of :filename, :maximum => 255
29
29
30 DATE_FORMATS = [
30 DATE_FORMATS = [
31 '%Y-%m-%d',
31 '%Y-%m-%d',
32 '%d/%m/%Y',
32 '%d/%m/%Y',
33 '%m/%d/%Y',
33 '%m/%d/%Y',
34 '%d.%m.%Y',
34 '%d.%m.%Y',
35 '%d-%m-%Y'
35 '%d-%m-%Y'
36 ]
36 ]
37
37
38 def initialize(*args)
38 def initialize(*args)
39 super
39 super
40 self.settings ||= {}
40 self.settings ||= {}
41 end
41 end
42
42
43 def file=(arg)
43 def file=(arg)
44 return unless arg.present? && arg.size > 0
44 return unless arg.present? && arg.size > 0
45
45
46 self.filename = generate_filename
46 self.filename = generate_filename
47 Redmine::Utils.save_upload(arg, filepath)
47 Redmine::Utils.save_upload(arg, filepath)
48 end
48 end
49
49
50 def set_default_settings
50 def set_default_settings
51 separator = lu(user, :general_csv_separator)
51 separator = lu(user, :general_csv_separator)
52 if file_exists?
52 if file_exists?
53 begin
53 begin
54 content = File.read(filepath, 256, "rb")
54 content = File.read(filepath, 256)
55 separator = [',', ';'].sort_by {|sep| content.count(sep) }.last
55 separator = [',', ';'].sort_by {|sep| content.count(sep) }.last
56 rescue Exception => e
56 rescue Exception => e
57 end
57 end
58 end
58 end
59 wrapper = '"'
59 wrapper = '"'
60 encoding = lu(user, :general_csv_encoding)
60 encoding = lu(user, :general_csv_encoding)
61
61
62 date_format = lu(user, "date.formats.default", :default => "foo")
62 date_format = lu(user, "date.formats.default", :default => "foo")
63 date_format = DATE_FORMATS.first unless DATE_FORMATS.include?(date_format)
63 date_format = DATE_FORMATS.first unless DATE_FORMATS.include?(date_format)
64
64
65 self.settings.merge!(
65 self.settings.merge!(
66 'separator' => separator,
66 'separator' => separator,
67 'wrapper' => wrapper,
67 'wrapper' => wrapper,
68 'encoding' => encoding,
68 'encoding' => encoding,
69 'date_format' => date_format
69 'date_format' => date_format
70 )
70 )
71 end
71 end
72
72
73 def to_param
73 def to_param
74 filename
74 filename
75 end
75 end
76
76
77 # Returns the full path of the file to import
77 # Returns the full path of the file to import
78 # It is stored in tmp/imports with a random hex as filename
78 # It is stored in tmp/imports with a random hex as filename
79 def filepath
79 def filepath
80 if filename.present? && filename =~ /\A[0-9a-f]+\z/
80 if filename.present? && filename =~ /\A[0-9a-f]+\z/
81 File.join(Rails.root, "tmp", "imports", filename)
81 File.join(Rails.root, "tmp", "imports", filename)
82 else
82 else
83 nil
83 nil
84 end
84 end
85 end
85 end
86
86
87 # Returns true if the file to import exists
87 # Returns true if the file to import exists
88 def file_exists?
88 def file_exists?
89 filepath.present? && File.exists?(filepath)
89 filepath.present? && File.exists?(filepath)
90 end
90 end
91
91
92 # Returns the headers as an array that
92 # Returns the headers as an array that
93 # can be used for select options
93 # can be used for select options
94 def columns_options(default=nil)
94 def columns_options(default=nil)
95 i = -1
95 i = -1
96 headers.map {|h| [h, i+=1]}
96 headers.map {|h| [h, i+=1]}
97 end
97 end
98
98
99 # Parses the file to import and updates the total number of items
99 # Parses the file to import and updates the total number of items
100 def parse_file
100 def parse_file
101 count = 0
101 count = 0
102 read_items {|row, i| count=i}
102 read_items {|row, i| count=i}
103 update_attribute :total_items, count
103 update_attribute :total_items, count
104 count
104 count
105 end
105 end
106
106
107 # Reads the items to import and yields the given block for each item
107 # Reads the items to import and yields the given block for each item
108 def read_items
108 def read_items
109 i = 0
109 i = 0
110 headers = true
110 headers = true
111 read_rows do |row|
111 read_rows do |row|
112 if i == 0 && headers
112 if i == 0 && headers
113 headers = false
113 headers = false
114 next
114 next
115 end
115 end
116 i+= 1
116 i+= 1
117 yield row, i if block_given?
117 yield row, i if block_given?
118 end
118 end
119 end
119 end
120
120
121 # Returns the count first rows of the file (including headers)
121 # Returns the count first rows of the file (including headers)
122 def first_rows(count=4)
122 def first_rows(count=4)
123 rows = []
123 rows = []
124 read_rows do |row|
124 read_rows do |row|
125 rows << row
125 rows << row
126 break if rows.size >= count
126 break if rows.size >= count
127 end
127 end
128 rows
128 rows
129 end
129 end
130
130
131 # Returns an array of headers
131 # Returns an array of headers
132 def headers
132 def headers
133 first_rows(1).first || []
133 first_rows(1).first || []
134 end
134 end
135
135
136 # Returns the mapping options
136 # Returns the mapping options
137 def mapping
137 def mapping
138 settings['mapping'] || {}
138 settings['mapping'] || {}
139 end
139 end
140
140
141 # Imports items and returns the position of the last processed item
141 # Imports items and returns the position of the last processed item
142 def run(options={})
142 def run(options={})
143 max_items = options[:max_items]
143 max_items = options[:max_items]
144 max_time = options[:max_time]
144 max_time = options[:max_time]
145 current = 0
145 current = 0
146 imported = 0
146 imported = 0
147 resume_after = items.maximum(:position) || 0
147 resume_after = items.maximum(:position) || 0
148 interrupted = false
148 interrupted = false
149 started_on = Time.now
149 started_on = Time.now
150
150
151 read_items do |row, position|
151 read_items do |row, position|
152 if (max_items && imported >= max_items) || (max_time && Time.now >= started_on + max_time)
152 if (max_items && imported >= max_items) || (max_time && Time.now >= started_on + max_time)
153 interrupted = true
153 interrupted = true
154 break
154 break
155 end
155 end
156 if position > resume_after
156 if position > resume_after
157 item = items.build
157 item = items.build
158 item.position = position
158 item.position = position
159
159
160 if object = build_object(row)
160 if object = build_object(row)
161 if object.save
161 if object.save
162 item.obj_id = object.id
162 item.obj_id = object.id
163 else
163 else
164 item.message = object.errors.full_messages.join("\n")
164 item.message = object.errors.full_messages.join("\n")
165 end
165 end
166 end
166 end
167
167
168 item.save!
168 item.save!
169 imported += 1
169 imported += 1
170 end
170 end
171 current = position
171 current = position
172 end
172 end
173
173
174 if imported == 0 || interrupted == false
174 if imported == 0 || interrupted == false
175 if total_items.nil?
175 if total_items.nil?
176 update_attribute :total_items, current
176 update_attribute :total_items, current
177 end
177 end
178 update_attribute :finished, true
178 update_attribute :finished, true
179 remove_file
179 remove_file
180 end
180 end
181
181
182 current
182 current
183 end
183 end
184
184
185 def unsaved_items
185 def unsaved_items
186 items.where(:obj_id => nil)
186 items.where(:obj_id => nil)
187 end
187 end
188
188
189 def saved_items
189 def saved_items
190 items.where("obj_id IS NOT NULL")
190 items.where("obj_id IS NOT NULL")
191 end
191 end
192
192
193 private
193 private
194
194
195 def read_rows
195 def read_rows
196 return unless file_exists?
196 return unless file_exists?
197
197
198 csv_options = {:headers => false}
198 csv_options = {:headers => false}
199 csv_options[:encoding] = settings['encoding'].to_s.presence || 'UTF-8'
199 csv_options[:encoding] = settings['encoding'].to_s.presence || 'UTF-8'
200 separator = settings['separator'].to_s
200 separator = settings['separator'].to_s
201 csv_options[:col_sep] = separator if separator.size == 1
201 csv_options[:col_sep] = separator if separator.size == 1
202 wrapper = settings['wrapper'].to_s
202 wrapper = settings['wrapper'].to_s
203 csv_options[:quote_char] = wrapper if wrapper.size == 1
203 csv_options[:quote_char] = wrapper if wrapper.size == 1
204
204
205 CSV.foreach(filepath, csv_options) do |row|
205 CSV.foreach(filepath, csv_options) do |row|
206 yield row if block_given?
206 yield row if block_given?
207 end
207 end
208 end
208 end
209
209
210 def row_value(row, key)
210 def row_value(row, key)
211 if index = mapping[key].presence
211 if index = mapping[key].presence
212 row[index.to_i].presence
212 row[index.to_i].presence
213 end
213 end
214 end
214 end
215
215
216 def row_date(row, key)
216 def row_date(row, key)
217 if s = row_value(row, key)
217 if s = row_value(row, key)
218 format = settings['date_format']
218 format = settings['date_format']
219 format = DATE_FORMATS.first unless DATE_FORMATS.include?(format)
219 format = DATE_FORMATS.first unless DATE_FORMATS.include?(format)
220 Date.strptime(s, format) rescue s
220 Date.strptime(s, format) rescue s
221 end
221 end
222 end
222 end
223
223
224 # Builds a record for the given row and returns it
224 # Builds a record for the given row and returns it
225 # To be implemented by subclasses
225 # To be implemented by subclasses
226 def build_object(row)
226 def build_object(row)
227 end
227 end
228
228
229 # Generates a filename used to store the import file
229 # Generates a filename used to store the import file
230 def generate_filename
230 def generate_filename
231 Redmine::Utils.random_hex(16)
231 Redmine::Utils.random_hex(16)
232 end
232 end
233
233
234 # Deletes the import file
234 # Deletes the import file
235 def remove_file
235 def remove_file
236 if file_exists?
236 if file_exists?
237 begin
237 begin
238 File.delete filepath
238 File.delete filepath
239 rescue Exception => e
239 rescue Exception => e
240 logger.error "Unable to delete file #{filepath}: #{e.message}" if logger
240 logger.error "Unable to delete file #{filepath}: #{e.message}" if logger
241 end
241 end
242 end
242 end
243 end
243 end
244
244
245 # Returns true if value is a string that represents a true value
245 # Returns true if value is a string that represents a true value
246 def yes?(value)
246 def yes?(value)
247 value == lu(user, :general_text_yes) || value == '1'
247 value == lu(user, :general_text_yes) || value == '1'
248 end
248 end
249 end
249 end
General Comments 0
You need to be logged in to leave comments. Login now