##// END OF EJS Templates
jro_apps » redmine
RSS
scm: replace invalid utf-8 sequences instead of stripping in displaying repository contents on Ruby 1.8....
Toshi MARUYAMA -
r5307:bb48d96d3037
parent child
Show More
Side by Side Unified
Context file:
r5307:bb48d96d3037 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,277 +1,283
1 1 # redMine - project management software
2 2 # Copyright (C) 2006 Jean-Philippe Lang
3 3 #
4 4 # This program is free software; you can redistribute it and/or
5 5 # modify it under the terms of the GNU General Public License
6 6 # as published by the Free Software Foundation; either version 2
7 7 # of the License, or (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software
16 16 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 17
18 18 require 'iconv'
19 19
20 20 module RepositoriesHelper
21 21 def format_revision(revision)
22 22 if revision.respond_to? :format_identifier
23 23 revision.format_identifier
24 24 else
25 25 revision.to_s
26 26 end
27 27 end
28 28
29 29 def truncate_at_line_break(text, length = 255)
30 30 if text
31 31 text.gsub(%r{^(.{#{length}}[^\n]*)\n.+$}m, '\\1...')
32 32 end
33 33 end
34 34
35 35 def render_properties(properties)
36 36 unless properties.nil? || properties.empty?
37 37 content = ''
38 38 properties.keys.sort.each do |property|
39 39 content << content_tag('li', "<b>#{h property}</b>: <span>#{h properties[property]}</span>")
40 40 end
41 41 content_tag('ul', content, :class => 'properties')
42 42 end
43 43 end
44 44
45 45 def render_changeset_changes
46 46 changes = @changeset.changes.find(:all, :limit => 1000, :order => 'path').collect do |change|
47 47 case change.action
48 48 when 'A'
49 49 # Detects moved/copied files
50 50 if !change.from_path.blank?
51 51 change.action = @changeset.changes.detect {|c| c.action == 'D' && c.path == change.from_path} ? 'R' : 'C'
52 52 end
53 53 change
54 54 when 'D'
55 55 @changeset.changes.detect {|c| c.from_path == change.path} ? nil : change
56 56 else
57 57 change
58 58 end
59 59 end.compact
60 60
61 61 tree = { }
62 62 changes.each do |change|
63 63 p = tree
64 64 dirs = change.path.to_s.split('/').select {|d| !d.blank?}
65 65 path = ''
66 66 dirs.each do |dir|
67 67 path += '/' + dir
68 68 p[:s] ||= {}
69 69 p = p[:s]
70 70 p[path] ||= {}
71 71 p = p[path]
72 72 end
73 73 p[:c] = change
74 74 end
75 75
76 76 render_changes_tree(tree[:s])
77 77 end
78 78
79 79 def render_changes_tree(tree)
80 80 return '' if tree.nil?
81 81
82 82 output = ''
83 83 output << '<ul>'
84 84 tree.keys.sort.each do |file|
85 85 style = 'change'
86 86 text = File.basename(h(file))
87 87 if s = tree[file][:s]
88 88 style << ' folder'
89 89 path_param = to_path_param(@repository.relative_path(file))
90 90 text = link_to(text, :controller => 'repositories',
91 91 :action => 'show',
92 92 :id => @project,
93 93 :path => path_param,
94 94 :rev => @changeset.identifier)
95 95 output << "<li class='#{style}'>#{text}</li>"
96 96 output << render_changes_tree(s)
97 97 elsif c = tree[file][:c]
98 98 style << " change-#{c.action}"
99 99 path_param = to_path_param(@repository.relative_path(c.path))
100 100 text = link_to(text, :controller => 'repositories',
101 101 :action => 'entry',
102 102 :id => @project,
103 103 :path => path_param,
104 104 :rev => @changeset.identifier) unless c.action == 'D'
105 105 text << " - #{c.revision}" unless c.revision.blank?
106 106 text << ' (' + link_to('diff', :controller => 'repositories',
107 107 :action => 'diff',
108 108 :id => @project,
109 109 :path => path_param,
110 110 :rev => @changeset.identifier) + ') ' if c.action == 'M'
111 111 text << ' ' + content_tag('span', c.from_path, :class => 'copied-from') unless c.from_path.blank?
112 112 output << "<li class='#{style}'>#{text}</li>"
113 113 end
114 114 end
115 115 output << '</ul>'
116 116 output
117 117 end
118 118
119 119 def to_utf8(str)
120 120 return str if str.nil?
121 121 str = to_utf8_internal(str)
122 122 if str.respond_to?(:force_encoding)
123 123 str.force_encoding('UTF-8')
124 124 end
125 125 str
126 126 end
127 127
128 128 def to_utf8_internal(str)
129 129 return str if str.nil?
130 130 if str.respond_to?(:force_encoding)
131 131 str.force_encoding('ASCII-8BIT')
132 132 end
133 133 return str if str.empty?
134 134 return str if /\A[\r\n\t\x20-\x7e]*\Z/n.match(str) # for us-ascii
135 135 if str.respond_to?(:force_encoding)
136 136 str.force_encoding('UTF-8')
137 137 end
138 138 @encodings ||= Setting.repositories_encodings.split(',').collect(&:strip)
139 139 @encodings.each do |encoding|
140 140 begin
141 141 return Iconv.conv('UTF-8', encoding, str)
142 142 rescue Iconv::Failure
143 143 # do nothing here and try the next encoding
144 144 end
145 145 end
146 146 str = replace_invalid_utf8(str)
147 147 end
148 148 private :to_utf8_internal
149 149
150 150 def replace_invalid_utf8(str)
151 151 return str if str.nil?
152 152 if str.respond_to?(:force_encoding)
153 153 str.force_encoding('UTF-8')
154 154 if ! str.valid_encoding?
155 155 str = str.encode("US-ASCII", :invalid => :replace,
156 156 :undef => :replace, :replace => '?').encode("UTF-8")
157 157 end
158 158 else
159 # removes invalid UTF8 sequences
159 ic = Iconv.new('UTF-8', 'UTF-8')
160 txtar = ""
160 161 begin
161 str = Iconv.conv('UTF-8//IGNORE', 'UTF-8', str + ' ')[0..-3]
162 rescue Iconv::InvalidEncoding
163 # "UTF-8//IGNORE" is not supported on some OS
164 end
162 txtar += ic.iconv(str)
163 rescue Iconv::IllegalSequence
164 txtar += $!.success
165 str = '?' + $!.failed[1,$!.failed.length]
166 retry
167 rescue
168 txtar += $!.success
169 end
170 str = txtar
165 171 end
166 172 str
167 173 end
168 174
169 175 def repository_field_tags(form, repository)
170 176 method = repository.class.name.demodulize.underscore + "_field_tags"
171 177 if repository.is_a?(Repository) &&
172 178 respond_to?(method) && method != 'repository_field_tags'
173 179 send(method, form, repository)
174 180 end
175 181 end
176 182
177 183 def scm_select_tag(repository)
178 184 scm_options = [["--- #{l(:actionview_instancetag_blank_option)} ---", '']]
179 185 Redmine::Scm::Base.all.each do |scm|
180 186 if Setting.enabled_scm.include?(scm) ||
181 187 (repository && repository.class.name.demodulize == scm)
182 188 scm_options << ["Repository::#{scm}".constantize.scm_name, scm]
183 189 end
184 190 end
185 191 select_tag('repository_scm',
186 192 options_for_select(scm_options, repository.class.name.demodulize),
187 193 :disabled => (repository && !repository.new_record?),
188 194 :onchange => remote_function(
189 195 :url => {
190 196 :controller => 'repositories',
191 197 :action => 'edit',
192 198 :id => @project
193 199 },
194 200 :method => :get,
195 201 :with => "Form.serialize(this.form)")
196 202 )
197 203 end
198 204
199 205 def with_leading_slash(path)
200 206 path.to_s.starts_with?('/') ? path : "/#{path}"
201 207 end
202 208
203 209 def without_leading_slash(path)
204 210 path.gsub(%r{^/+}, '')
205 211 end
206 212
207 213 def subversion_field_tags(form, repository)
208 214 content_tag('p', form.text_field(:url, :size => 60, :required => true,
209 215 :disabled => (repository && !repository.root_url.blank?)) +
210 216 '<br />(file:///, http://, https://, svn://, svn+[tunnelscheme]://)') +
211 217 content_tag('p', form.text_field(:login, :size => 30)) +
212 218 content_tag('p', form.password_field(
213 219 :password, :size => 30, :name => 'ignore',
214 220 :value => ((repository.new_record? || repository.password.blank?) ? '' : ('x'*15)),
215 221 :onfocus => "this.value=''; this.name='repository[password]';",
216 222 :onchange => "this.name='repository[password]';"))
217 223 end
218 224
219 225 def darcs_field_tags(form, repository)
220 226 content_tag('p', form.text_field(:url, :label => 'Root directory',
221 227 :size => 60, :required => true,
222 228 :disabled => (repository && !repository.new_record?))) +
223 229 content_tag('p', form.select(:log_encoding, [nil] + Setting::ENCODINGS,
224 230 :label => 'Commit messages encoding', :required => true))
225 231 end
226 232
227 233 def mercurial_field_tags(form, repository)
228 234 content_tag('p', form.text_field(:url, :label => 'Root directory',
229 235 :size => 60, :required => true,
230 236 :disabled => (repository && !repository.root_url.blank?)) +
231 237 '<br />Local repository (e.g. /hgrepo, c:\hgrepo)' ) +
232 238 content_tag('p', form.select(
233 239 :path_encoding, [nil] + Setting::ENCODINGS,
234 240 :label => 'Path encoding') +
235 241 '<br />Default: UTF-8')
236 242 end
237 243
238 244 def git_field_tags(form, repository)
239 245 content_tag('p', form.text_field(:url, :label => 'Path to repository',
240 246 :size => 60, :required => true,
241 247 :disabled => (repository && !repository.root_url.blank?)) +
242 248 '<br />Bare and local repository (e.g. /gitrepo, c:\gitrepo)') +
243 249 content_tag('p', form.select(
244 250 :path_encoding, [nil] + Setting::ENCODINGS,
245 251 :label => 'Path encoding') +
246 252 '<br />Default: UTF-8')
247 253 end
248 254
249 255 def cvs_field_tags(form, repository)
250 256 content_tag('p', form.text_field(:root_url,
251 257 :label => 'CVSROOT', :size => 60, :required => true,
252 258 :disabled => !repository.new_record?)) +
253 259 content_tag('p', form.text_field(:url, :label => 'Module',
254 260 :size => 30, :required => true,
255 261 :disabled => !repository.new_record?)) +
256 262 content_tag('p', form.select(:log_encoding, [nil] + Setting::ENCODINGS,
257 263 :label => 'Commit messages encoding', :required => true))
258 264 end
259 265
260 266 def bazaar_field_tags(form, repository)
261 267 content_tag('p', form.text_field(:url, :label => 'Root directory',
262 268 :size => 60, :required => true,
263 269 :disabled => (repository && !repository.new_record?))) +
264 270 content_tag('p', form.select(:log_encoding, [nil] + Setting::ENCODINGS,
265 271 :label => 'Commit messages encoding', :required => true))
266 272 end
267 273
268 274 def filesystem_field_tags(form, repository)
269 275 content_tag('p', form.text_field(:url, :label => 'Root directory',
270 276 :size => 60, :required => true,
271 277 :disabled => (repository && !repository.root_url.blank?))) +
272 278 content_tag('p', form.select(
273 279 :path_encoding, [nil] + Setting::ENCODINGS,
274 280 :label => 'Path encoding') +
275 281 '<br />Default: UTF-8')
276 282 end
277 283 end
Show file before | Show file after | Hide comments
@@ -1,105 +1,103
1 1 # Redmine - project management software
2 2 # Copyright (C) 2006-2011 Jean-Philippe Lang
3 3 #
4 4 # This program is free software; you can redistribute it and/or
5 5 # modify it under the terms of the GNU General Public License
6 6 # as published by the Free Software Foundation; either version 2
7 7 # of the License, or (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software
16 16 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 17
18 18 require File.expand_path('../../../test_helper', __FILE__)
19 19
20 20 class RepositoryHelperTest < HelperTestCase
21 21 include RepositoriesHelper
22 22
23 23 def test_from_latin1_to_utf8
24 24 with_settings :repositories_encodings => 'UTF-8,ISO-8859-1' do
25 25 s1 = "Texte encod\xc3\xa9"
26 26 s2 = "Texte encod\xe9"
27 27 s3 = s2.dup
28 28 if s1.respond_to?(:force_encoding)
29 29 s1.force_encoding("UTF-8")
30 30 s2.force_encoding("ASCII-8BIT")
31 31 s3.force_encoding("UTF-8")
32 32 end
33 33 assert_equal s1, to_utf8(s2)
34 34 assert_equal s1, to_utf8(s3)
35 35 end
36 36 end
37 37
38 38 def test_from_euc_jp_to_utf8
39 39 with_settings :repositories_encodings => 'UTF-8,EUC-JP' do
40 40 s1 = "\xe3\x83\xac\xe3\x83\x83\xe3\x83\x89\xe3\x83\x9e\xe3\x82\xa4\xe3\x83\xb3"
41 41 s2 = "\xa5\xec\xa5\xc3\xa5\xc9\xa5\xde\xa5\xa4\xa5\xf3"
42 42 s3 = s2.dup
43 43 if s1.respond_to?(:force_encoding)
44 44 s1.force_encoding("UTF-8")
45 45 s2.force_encoding("ASCII-8BIT")
46 46 s3.force_encoding("UTF-8")
47 47 end
48 48 assert_equal s1, to_utf8(s2)
49 49 assert_equal s1, to_utf8(s3)
50 50 end
51 51 end
52 52
53 53 def test_to_utf8_should_be_converted_all_latin1_to_utf8
54 54 with_settings :repositories_encodings => 'ISO-8859-1' do
55 55 s1 = "\xc3\x82\xc2\x80"
56 56 s2 = "\xC2\x80"
57 57 s3 = s2.dup
58 58 if s1.respond_to?(:force_encoding)
59 59 s1.force_encoding("UTF-8")
60 60 s2.force_encoding("ASCII-8BIT")
61 61 s3.force_encoding("UTF-8")
62 62 end
63 63 assert_equal s1, to_utf8(s2)
64 64 assert_equal s1, to_utf8(s3)
65 65 end
66 66 end
67 67
68 68 def test_to_utf8_blank_string
69 69 assert_equal "", to_utf8("")
70 70 assert_equal nil, to_utf8(nil)
71 71 end
72 72
73 73 def test_to_utf8_returns_ascii_as_utf8
74 74 s1 = "ASCII"
75 75 s2 = s1.dup
76 76 if s1.respond_to?(:force_encoding)
77 77 s1.force_encoding("UTF-8")
78 78 s2.force_encoding("ISO-8859-1")
79 79 end
80 80 str1 = to_utf8(s1)
81 81 str2 = to_utf8(s2)
82 82 assert_equal s1, str1
83 83 assert_equal s1, str2
84 84 if s1.respond_to?(:force_encoding)
85 85 assert_equal "UTF-8", str1.encoding.to_s
86 86 assert_equal "UTF-8", str2.encoding.to_s
87 87 end
88 88 end
89 89
90 90 def test_to_utf8_invalid_utf8_sequences_should_be_stripped
91 91 with_settings :repositories_encodings => '' do
92 92 # s1 = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")
93 93 s1 = "Texte encod\xe9 en ISO-8859-1."
94 94 s1.force_encoding("ASCII-8BIT") if s1.respond_to?(:force_encoding)
95 95 str = to_utf8(s1)
96 96 if str.respond_to?(:force_encoding)
97 assert_equal "Texte encod? en ISO-8859-1.", str
98 97 assert str.valid_encoding?
99 98 assert_equal "UTF-8", str.encoding.to_s
100 else
101 assert_equal "Texte encod en ISO-8859-1.", str
102 99 end
100 assert_equal "Texte encod? en ISO-8859-1.", str
103 101 end
104 102 end
105 103 end
General Comments 0
You need to be logged in to leave comments. Login now