##// END OF EJS Templates
jro_apps » redmine
RSS
Remove invalid utf8 sequences from commit comments and author name (#4773)....
Jean-Philippe Lang -
r3352:46aa855572a2
parent child
Show More
Side by Side Unified
Context file:
r3352:46aa855572a2 Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,1
1 Texte encod� en ISO-8859-1. No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,190 +1,195
1 1 # Redmine - project management software
2 # Copyright (C) 2006-2008 Jean-Philippe Lang
2 # Copyright (C) 2006-2010 Jean-Philippe Lang
3 3 #
4 4 # This program is free software; you can redistribute it and/or
5 5 # modify it under the terms of the GNU General Public License
6 6 # as published by the Free Software Foundation; either version 2
7 7 # of the License, or (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software
16 16 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 17
18 18 require 'iconv'
19 19
20 20 class Changeset < ActiveRecord::Base
21 21 belongs_to :repository
22 22 belongs_to :user
23 23 has_many :changes, :dependent => :delete_all
24 24 has_and_belongs_to_many :issues
25 25
26 26 acts_as_event :title => Proc.new {|o| "#{l(:label_revision)} #{o.revision}" + (o.short_comments.blank? ? '' : (': ' + o.short_comments))},
27 27 :description => :long_comments,
28 28 :datetime => :committed_on,
29 29 :url => Proc.new {|o| {:controller => 'repositories', :action => 'revision', :id => o.repository.project, :rev => o.revision}}
30 30
31 31 acts_as_searchable :columns => 'comments',
32 32 :include => {:repository => :project},
33 33 :project_key => "#{Repository.table_name}.project_id",
34 34 :date_column => 'committed_on'
35 35
36 36 acts_as_activity_provider :timestamp => "#{table_name}.committed_on",
37 37 :author_key => :user_id,
38 38 :find_options => {:include => [:user, {:repository => :project}]}
39 39
40 40 validates_presence_of :repository_id, :revision, :committed_on, :commit_date
41 41 validates_uniqueness_of :revision, :scope => :repository_id
42 42 validates_uniqueness_of :scmid, :scope => :repository_id, :allow_nil => true
43 43
44 44 named_scope :visible, lambda {|*args| { :include => {:repository => :project},
45 45 :conditions => Project.allowed_to_condition(args.first || User.current, :view_changesets) } }
46 46
47 47 def revision=(r)
48 48 write_attribute :revision, (r.nil? ? nil : r.to_s)
49 49 end
50 50
51 51 def comments=(comment)
52 52 write_attribute(:comments, Changeset.normalize_comments(comment))
53 53 end
54 54
55 55 def committed_on=(date)
56 56 self.commit_date = date
57 57 super
58 58 end
59 59
60 def committer=(arg)
61 write_attribute(:committer, self.class.to_utf8(arg.to_s))
62 end
63
60 64 def project
61 65 repository.project
62 66 end
63 67
64 68 def author
65 69 user || committer.to_s.split('<').first
66 70 end
67 71
68 72 def before_create
69 73 self.user = repository.find_committer_user(committer)
70 74 end
71 75
72 76 def after_create
73 77 scan_comment_for_issue_ids
74 78 end
75 79 require 'pp'
76 80
77 81 def scan_comment_for_issue_ids
78 82 return if comments.blank?
79 83 # keywords used to reference issues
80 84 ref_keywords = Setting.commit_ref_keywords.downcase.split(",").collect(&:strip)
81 85 # keywords used to fix issues
82 86 fix_keywords = Setting.commit_fix_keywords.downcase.split(",").collect(&:strip)
83 87 # status and optional done ratio applied
84 88 fix_status = IssueStatus.find_by_id(Setting.commit_fix_status_id)
85 89 done_ratio = Setting.commit_fix_done_ratio.blank? ? nil : Setting.commit_fix_done_ratio.to_i
86 90
87 91 kw_regexp = (ref_keywords + fix_keywords).collect{|kw| Regexp.escape(kw)}.join("|")
88 92 return if kw_regexp.blank?
89 93
90 94 referenced_issues = []
91 95
92 96 if ref_keywords.delete('*')
93 97 # find any issue ID in the comments
94 98 target_issue_ids = []
95 99 comments.scan(%r{([\s\(\[,-]|^)#(\d+)(?=[[:punct:]]|\s|<|$)}).each { |m| target_issue_ids << m[1] }
96 100 referenced_issues += find_referenced_issues_by_id(target_issue_ids)
97 101 end
98 102
99 103 comments.scan(Regexp.new("(#{kw_regexp})[\s:]+(([\s,;&]*#?\\d+)+)", Regexp::IGNORECASE)).each do |match|
100 104 action = match[0]
101 105 target_issue_ids = match[1].scan(/\d+/)
102 106 target_issues = find_referenced_issues_by_id(target_issue_ids)
103 107 if fix_status && fix_keywords.include?(action.downcase)
104 108 # update status of issues
105 109 logger.debug "Issues fixed by changeset #{self.revision}: #{issue_ids.join(', ')}." if logger && logger.debug?
106 110 target_issues.each do |issue|
107 111 # the issue may have been updated by the closure of another one (eg. duplicate)
108 112 issue.reload
109 113 # don't change the status is the issue is closed
110 114 next if issue.status.is_closed?
111 115 csettext = "r#{self.revision}"
112 116 if self.scmid && (! (csettext =~ /^r[0-9]+$/))
113 117 csettext = "commit:\"#{self.scmid}\""
114 118 end
115 119 journal = issue.init_journal(user || User.anonymous, ll(Setting.default_language, :text_status_changed_by_changeset, csettext))
116 120 issue.status = fix_status
117 121 issue.done_ratio = done_ratio if done_ratio
118 122 Redmine::Hook.call_hook(:model_changeset_scan_commit_for_issue_ids_pre_issue_update,
119 123 { :changeset => self, :issue => issue })
120 124 issue.save
121 125 end
122 126 end
123 127 referenced_issues += target_issues
124 128 end
125 129
126 130 self.issues = referenced_issues.uniq
127 131 end
128 132
129 133 def short_comments
130 134 @short_comments || split_comments.first
131 135 end
132 136
133 137 def long_comments
134 138 @long_comments || split_comments.last
135 139 end
136 140
137 141 # Returns the previous changeset
138 142 def previous
139 143 @previous ||= Changeset.find(:first, :conditions => ['id < ? AND repository_id = ?', self.id, self.repository_id], :order => 'id DESC')
140 144 end
141 145
142 146 # Returns the next changeset
143 147 def next
144 148 @next ||= Changeset.find(:first, :conditions => ['id > ? AND repository_id = ?', self.id, self.repository_id], :order => 'id ASC')
145 149 end
146 150
147 151 # Strips and reencodes a commit log before insertion into the database
148 152 def self.normalize_comments(str)
149 153 to_utf8(str.to_s.strip)
150 154 end
151 155
152 156 # Creates a new Change from it's common parameters
153 157 def create_change(change)
154 158 Change.create(:changeset => self,
155 159 :action => change[:action],
156 160 :path => change[:path],
157 161 :from_path => change[:from_path],
158 162 :from_revision => change[:from_revision])
159 163 end
160 164
161 165 private
162 166
163 167 # Finds issues that can be referenced by the commit message
164 168 # i.e. issues that belong to the repository project, a subproject or a parent project
165 169 def find_referenced_issues_by_id(ids)
166 170 Issue.find_all_by_id(ids, :include => :project).select {|issue|
167 171 project == issue.project || project.is_ancestor_of?(issue.project) || project.is_descendant_of?(issue.project)
168 172 }
169 173 end
170 174
171 175 def split_comments
172 176 comments =~ /\A(.+?)\r?\n(.*)$/m
173 177 @short_comments = $1 || comments
174 178 @long_comments = $2.to_s.strip
175 179 return @short_comments, @long_comments
176 180 end
177 181
178 182 def self.to_utf8(str)
179 183 return str if /\A[\r\n\t\x20-\x7e]*\Z/n.match(str) # for us-ascii
180 184 encoding = Setting.commit_logs_encoding.to_s.strip
181 185 unless encoding.blank? || encoding == 'UTF-8'
182 186 begin
183 return Iconv.conv('UTF-8', encoding, str)
187 str = Iconv.conv('UTF-8', encoding, str)
184 188 rescue Iconv::Failure
185 189 # do nothing here
186 190 end
187 191 end
188 str
192 # removes invalid UTF8 sequences
193 Iconv.conv('UTF-8//IGNORE', 'UTF-8', str + ' ')[0..-3]
189 194 end
190 195 end
Show file before | Show file after | Hide comments
@@ -1,120 +1,136
1 # redMine - project management software
2 # Copyright (C) 2006-2007 Jean-Philippe Lang
1 # encoding: utf-8
2 #
3 # Redmine - project management software
4 # Copyright (C) 2006-2010 Jean-Philippe Lang
3 5 #
4 6 # This program is free software; you can redistribute it and/or
5 7 # modify it under the terms of the GNU General Public License
6 8 # as published by the Free Software Foundation; either version 2
7 9 # of the License, or (at your option) any later version.
8 10 #
9 11 # This program is distributed in the hope that it will be useful,
10 12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 14 # GNU General Public License for more details.
13 15 #
14 16 # You should have received a copy of the GNU General Public License
15 17 # along with this program; if not, write to the Free Software
16 18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 19
18 20 require File.dirname(__FILE__) + '/../test_helper'
19 21
20 22 class ChangesetTest < ActiveSupport::TestCase
21 23 fixtures :projects, :repositories, :issues, :issue_statuses, :changesets, :changes, :issue_categories, :enumerations, :custom_fields, :custom_values, :users, :members, :member_roles, :trackers
22 24
23 25 def setup
24 26 end
25 27
26 28 def test_ref_keywords_any
27 29 ActionMailer::Base.deliveries.clear
28 30 Setting.commit_fix_status_id = IssueStatus.find(:first, :conditions => ["is_closed = ?", true]).id
29 31 Setting.commit_fix_done_ratio = '90'
30 32 Setting.commit_ref_keywords = '*'
31 33 Setting.commit_fix_keywords = 'fixes , closes'
32 34
33 35 c = Changeset.new(:repository => Project.find(1).repository,
34 36 :committed_on => Time.now,
35 37 :comments => 'New commit (#2). Fixes #1')
36 38 c.scan_comment_for_issue_ids
37 39
38 40 assert_equal [1, 2], c.issue_ids.sort
39 41 fixed = Issue.find(1)
40 42 assert fixed.closed?
41 43 assert_equal 90, fixed.done_ratio
42 44 assert_equal 1, ActionMailer::Base.deliveries.size
43 45 end
44 46
45 47 def test_ref_keywords_any_line_start
46 48 Setting.commit_ref_keywords = '*'
47 49
48 50 c = Changeset.new(:repository => Project.find(1).repository,
49 51 :committed_on => Time.now,
50 52 :comments => '#1 is the reason of this commit')
51 53 c.scan_comment_for_issue_ids
52 54
53 55 assert_equal [1], c.issue_ids.sort
54 56 end
55 57
56 58 def test_ref_keywords_allow_brackets_around_a_issue_number
57 59 Setting.commit_ref_keywords = '*'
58 60
59 61 c = Changeset.new(:repository => Project.find(1).repository,
60 62 :committed_on => Time.now,
61 63 :comments => '[#1] Worked on this issue')
62 64 c.scan_comment_for_issue_ids
63 65
64 66 assert_equal [1], c.issue_ids.sort
65 67 end
66 68
67 69 def test_ref_keywords_allow_brackets_around_multiple_issue_numbers
68 70 Setting.commit_ref_keywords = '*'
69 71
70 72 c = Changeset.new(:repository => Project.find(1).repository,
71 73 :committed_on => Time.now,
72 74 :comments => '[#1 #2, #3] Worked on these')
73 75 c.scan_comment_for_issue_ids
74 76
75 77 assert_equal [1,2,3], c.issue_ids.sort
76 78 end
77 79
78 80 def test_commit_referencing_a_subproject_issue
79 81 c = Changeset.new(:repository => Project.find(1).repository,
80 82 :committed_on => Time.now,
81 83 :comments => 'refs #5, a subproject issue')
82 84 c.scan_comment_for_issue_ids
83 85
84 86 assert_equal [5], c.issue_ids.sort
85 87 assert c.issues.first.project != c.project
86 88 end
87 89
88 90 def test_commit_referencing_a_parent_project_issue
89 91 # repository of child project
90 92 r = Repository::Subversion.create!(:project => Project.find(3), :url => 'svn://localhost/test')
91 93
92 94 c = Changeset.new(:repository => r,
93 95 :committed_on => Time.now,
94 96 :comments => 'refs #2, an issue of a parent project')
95 97 c.scan_comment_for_issue_ids
96 98
97 99 assert_equal [2], c.issue_ids.sort
98 100 assert c.issues.first.project != c.project
99 101 end
100 102
101 103 def test_previous
102 104 changeset = Changeset.find_by_revision('3')
103 105 assert_equal Changeset.find_by_revision('2'), changeset.previous
104 106 end
105 107
106 108 def test_previous_nil
107 109 changeset = Changeset.find_by_revision('1')
108 110 assert_nil changeset.previous
109 111 end
110 112
111 113 def test_next
112 114 changeset = Changeset.find_by_revision('2')
113 115 assert_equal Changeset.find_by_revision('3'), changeset.next
114 116 end
115 117
116 118 def test_next_nil
117 119 changeset = Changeset.find_by_revision('10')
118 120 assert_nil changeset.next
119 121 end
122
123 def test_comments_should_be_converted_to_utf8
124 with_settings :commit_logs_encoding => 'ISO-8859-1' do
125 c = Changeset.new
126 c.comments = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")
127 assert_equal "Texte encodé en ISO-8859-1.", c.comments
128 end
129 end
130
131 def test_invalid_utf8_sequences_in_comments_should_be_stripped
132 c = Changeset.new
133 c.comments = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")
134 assert_equal "Texte encod en ISO-8859-1.", c.comments
135 end
120 136 end
General Comments 0
You need to be logged in to leave comments. Login now