jro_apps/redmine Commit - r15891:b2c0ea2c3e15

1

# Redmine - project management software

1

# Redmine - project management software

2

3

#

3

#

4

# This program is free software; you can redistribute it and/or

4

# This program is free software; you can redistribute it and/or

5

# modify it under the terms of the GNU General Public License

5

# modify it under the terms of the GNU General Public License

6

# as published by the Free Software Foundation; either version 2

6

# as published by the Free Software Foundation; either version 2

7

# of the License, or (at your option) any later version.

7

# of the License, or (at your option) any later version.

8

#

8

#

9

# This program is distributed in the hope that it will be useful,

9

# This program is distributed in the hope that it will be useful,

10

# but WITHOUT ANY WARRANTY; without even the implied warranty of

10

# but WITHOUT ANY WARRANTY; without even the implied warranty of

11

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

11

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

12

# GNU General Public License for more details.

12

# GNU General Public License for more details.

13

#

13

#

14

# You should have received a copy of the GNU General Public License

14

# You should have received a copy of the GNU General Public License

15

# along with this program; if not, write to the Free Software

15

# along with this program; if not, write to the Free Software

16

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

16

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

17

18

require File.expand_path('../../../../test_helper', __FILE__)

18

require File.expand_path('../../../../test_helper', __FILE__)

19

20

class Redmine::CodesetUtilTest < ActiveSupport::TestCase

20

class Redmine::CodesetUtilTest < ActiveSupport::TestCase

21

22

def test_to_utf8_by_setting_from_latin1

22

def test_to_utf8_by_setting_from_latin1

23

with_settings :repositories_encodings => 'UTF-8,ISO-8859-1' do

23

with_settings :repositories_encodings => 'UTF-8,ISO-8859-1' do

24

s1 = "Texte encod\xc3\xa9".force_encoding("UTF-8")

24

s1 = "Texte encod\xc3\xa9".force_encoding("UTF-8")

25

s2 = "Texte encod\xe9".force_encoding("ASCII-8BIT")

25

s2 = "Texte encod\xe9".force_encoding("ASCII-8BIT")

26

s3 = s2.dup.force_encoding("UTF-8")

26

s3 = s2.dup.force_encoding("UTF-8")

27

assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s2)

27

assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s2)

28

assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s3)

28

assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s3)

29

end

29

end

30

end

30

end

31

32

def test_to_utf8_by_setting_from_euc_jp

32

def test_to_utf8_by_setting_from_euc_jp

33

with_settings :repositories_encodings => 'UTF-8,EUC-JP' do

33

with_settings :repositories_encodings => 'UTF-8,EUC-JP' do

34

s1 = "\xe3\x83\xac\xe3\x83\x83\xe3\x83\x89\xe3\x83\x9e\xe3\x82\xa4\xe3\x83\xb3".force_encoding("UTF-8")

34

s1 = "\xe3\x83\xac\xe3\x83\x83\xe3\x83\x89\xe3\x83\x9e\xe3\x82\xa4\xe3\x83\xb3".force_encoding("UTF-8")

35

s2 = "\xa5\xec\xa5\xc3\xa5\xc9\xa5\xde\xa5\xa4\xa5\xf3".force_encoding("ASCII-8BIT")

35

s2 = "\xa5\xec\xa5\xc3\xa5\xc9\xa5\xde\xa5\xa4\xa5\xf3".force_encoding("ASCII-8BIT")

36

s3 = s2.dup.force_encoding("UTF-8")

36

s3 = s2.dup.force_encoding("UTF-8")

37

assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s2)

37

assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s2)

38

assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s3)

38

assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s3)

39

end

39

end

40

end

40

end

41

42

def test_to_utf8_by_setting_should_be_converted_all_latin1

42

def test_to_utf8_by_setting_should_be_converted_all_latin1

43

with_settings :repositories_encodings => 'ISO-8859-1' do

43

with_settings :repositories_encodings => 'ISO-8859-1' do

44

s1 = "\xc3\x82\xc2\x80".force_encoding("UTF-8")

44

s1 = "\xc3\x82\xc2\x80".force_encoding("UTF-8")

45

s2 = "\xC2\x80".force_encoding("ASCII-8BIT")

45

s2 = "\xC2\x80".force_encoding("ASCII-8BIT")

46

s3 = s2.dup.force_encoding("UTF-8")

46

s3 = s2.dup.force_encoding("UTF-8")

47

assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s2)

47

assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s2)

48

assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s3)

48

assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s3)

49

end

49

end

50

end

50

end

51

52

def test_to_utf8_by_setting_blank_string

52

def test_to_utf8_by_setting_blank_string

53

assert_equal "", Redmine::CodesetUtil.to_utf8_by_setting("")

53

assert_equal "", Redmine::CodesetUtil.to_utf8_by_setting("")

54

assert_nil Redmine::CodesetUtil.to_utf8_by_setting(nil)

54

assert_nil Redmine::CodesetUtil.to_utf8_by_setting(nil)

55

end

55

end

56

57

def test_to_utf8_by_setting_returns_ascii_as_utf8

57

def test_to_utf8_by_setting_returns_ascii_as_utf8

58

s1 = "ASCII".force_encoding("UTF-8")

58

s1 = "ASCII".force_encoding("UTF-8")

59

s2 = s1.dup.force_encoding("ISO-8859-1")

59

s2 = s1.dup.force_encoding("ISO-8859-1")

60

str1 = Redmine::CodesetUtil.to_utf8_by_setting(s1)

60

str1 = Redmine::CodesetUtil.to_utf8_by_setting(s1)

61

str2 = Redmine::CodesetUtil.to_utf8_by_setting(s2)

61

str2 = Redmine::CodesetUtil.to_utf8_by_setting(s2)

62

assert_equal s1, str1

62

assert_equal s1, str1

63

assert_equal s1, str2

63

assert_equal s1, str2

64

assert_equal "UTF-8", str1.encoding.to_s

64

assert_equal "UTF-8", str1.encoding.to_s

65

assert_equal "UTF-8", str2.encoding.to_s

65

assert_equal "UTF-8", str2.encoding.to_s

66

end

66

end

67

68

def test_to_utf8_by_setting_invalid_utf8_sequences_should_be_stripped

68

def test_to_utf8_by_setting_invalid_utf8_sequences_should_be_stripped

69

with_settings :repositories_encodings => '' do

69

with_settings :repositories_encodings => '' do

70

# s1 = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")

70

# s1 = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")

71

s1 = "Texte encod\xe9 en ISO-8859-1.".force_encoding("ASCII-8BIT")

71

s1 = "Texte encod\xe9 en ISO-8859-1.".force_encoding("ASCII-8BIT")

72

str = Redmine::CodesetUtil.to_utf8_by_setting(s1)

72

str = Redmine::CodesetUtil.to_utf8_by_setting(s1)

73

assert str.valid_encoding?

73

assert str.valid_encoding?

74

assert_equal "UTF-8", str.encoding.to_s

74

assert_equal "UTF-8", str.encoding.to_s

75

assert_equal "Texte encod? en ISO-8859-1.", str

75

assert_equal "Texte encod? en ISO-8859-1.", str

76

end

76

end

77

end

77

end

78

79

def test_to_utf8_by_setting_invalid_utf8_sequences_should_be_stripped_ja_jis

79

def test_to_utf8_by_setting_invalid_utf8_sequences_should_be_stripped_ja_jis

80

with_settings :repositories_encodings => 'ISO-2022-JP' do

80

with_settings :repositories_encodings => 'ISO-2022-JP' do

81

s1 = "test\xb5\xfetest\xb5\xfe".force_encoding("ASCII-8BIT")

81

s1 = "test\xb5\xfetest\xb5\xfe".force_encoding("ASCII-8BIT")

82

str = Redmine::CodesetUtil.to_utf8_by_setting(s1)

82

str = Redmine::CodesetUtil.to_utf8_by_setting(s1)

83

assert str.valid_encoding?

83

assert str.valid_encoding?

84

assert_equal "UTF-8", str.encoding.to_s

84

assert_equal "UTF-8", str.encoding.to_s

85

assert_equal "test??test??", str

85

assert_equal "test??test??", str

86

end

86

end

87

end

87

end

88

89

test "#replace_invalid_utf8 should replace invalid utf8" do

89

test "#replace_invalid_utf8 should replace invalid utf8" do

90

s1 = "\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xE3\x81\xFF".force_encoding("UTF-8")

90

s1 = "\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xE3\x81\xFF".force_encoding("UTF-8")

91

s2 = Redmine::CodesetUtil.replace_invalid_utf8(s1)

91

s2 = Redmine::CodesetUtil.replace_invalid_utf8(s1)

92

assert s2.valid_encoding?

92

assert s2.valid_encoding?

93

assert_equal "UTF-8", s2.encoding.to_s

93

assert_equal "UTF-8", s2.encoding.to_s

94

assert_equal "??????", s2

94

assert_equal "\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1??".force_encoding("UTF-8"), s2

95

end

95

end

96

97

test "#to_utf8 should replace invalid non utf8" do

97

test "#to_utf8 should replace invalid non utf8" do

98

s1 = "\xa4\xb3\xa4\xf3\xa4\xcb\xa4\xc1\xa4".force_encoding("EUC-JP")

98

s1 = "\xa4\xb3\xa4\xf3\xa4\xcb\xa4\xc1\xa4".force_encoding("EUC-JP")

99

s2 = Redmine::CodesetUtil.to_utf8(s1, "EUC-JP")

99

s2 = Redmine::CodesetUtil.to_utf8(s1, "EUC-JP")

100

assert s2.valid_encoding?

100

assert s2.valid_encoding?

101

assert_equal "UTF-8", s2.encoding.to_s

101

assert_equal "UTF-8", s2.encoding.to_s

102

assert_equal "\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1?".force_encoding("UTF-8"), s2

102

assert_equal "\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1?".force_encoding("UTF-8"), s2

103

end

103

end

104

end

104

end

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages

             module Redmine
               module CodesetUtil
                 def self.replace_invalid_utf8(str)
                   return str if str.nil?
                   str.force_encoding('UTF-8')
                   if ! str.valid_encoding?
-                    str = str.encode("US-ASCII", :invalid => :replace,
+                    str = str.encode("UTF-16LE", :invalid => :replace,
                           :undef => :replace, :replace => '?').encode("UTF-8")
                   end
                   str
                 end
                 def self.to_utf8(str, encoding)
                   return str if str.nil?
                   str.force_encoding("ASCII-8BIT")
                   if str.empty?
                     str.force_encoding("UTF-8")
                     return str
                   end
                   enc = encoding.blank? ? "UTF-8" : encoding
                   if enc.upcase != "UTF-8"
                     str.force_encoding(enc)
                     str = str.encode("UTF-8", :invalid => :replace,
                           :undef => :replace, :replace => '?')
                   else
                     str = replace_invalid_utf8(str)
                   end
                   str
                 end
                 def self.to_utf8_by_setting(str)
                   return str if str.nil?
                   self.to_utf8_by_setting_internal(str).force_encoding('UTF-8')
                 end
                 def self.to_utf8_by_setting_internal(str)
                   return str if str.nil?
                   str.force_encoding('ASCII-8BIT')
                   return str if str.empty?
                   return str if /\A[\r\n\t\x20-\x7e]*\Z/n.match(str) # for us-ascii
                   str.force_encoding('UTF-8')
                   encodings = Setting.repositories_encodings.split(',').collect(&:strip)
                   encodings.each do |encoding|
                     begin
                       str.force_encoding(encoding)
                       utf8 = str.encode('UTF-8')
                       return utf8 if utf8.valid_encoding?
                     rescue
                       # do nothing here and try the next encoding
                     end
                   end
                   self.replace_invalid_utf8(str).force_encoding('UTF-8')
                 end
                 def self.from_utf8(str, encoding)
                   str ||= ''
                   str.force_encoding('UTF-8')
                   if encoding.upcase != 'UTF-8'
                     str = str.encode(encoding, :invalid => :replace,
                                      :undef => :replace, :replace => '?')
                   else
                     str = self.replace_invalid_utf8(str)
                   end
                 end
               end
             end

             # Redmine - project management software
             # Copyright (C) 2006-2016  Jean-Philippe Lang
             #
             # This program is free software; you can redistribute it and/or
             # modify it under the terms of the GNU General Public License
             # as published by the Free Software Foundation; either version 2
             # of the License, or (at your option) any later version.
             #
             # This program is distributed in the hope that it will be useful,
             # but WITHOUT ANY WARRANTY; without even the implied warranty of
             # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
             # GNU General Public License for more details.
             #
             # You should have received a copy of the GNU General Public License
             # along with this program; if not, write to the Free Software
             # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
             require File.expand_path('../../../../test_helper', __FILE__)
             class Redmine::CodesetUtilTest < ActiveSupport::TestCase
               def test_to_utf8_by_setting_from_latin1
                 with_settings :repositories_encodings => 'UTF-8,ISO-8859-1' do
                   s1 = "Texte encod\xc3\xa9".force_encoding("UTF-8")
                   s2 = "Texte encod\xe9".force_encoding("ASCII-8BIT")
                   s3 = s2.dup.force_encoding("UTF-8")
                   assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s2)
                   assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s3)
                 end
               end
               def test_to_utf8_by_setting_from_euc_jp
                 with_settings :repositories_encodings => 'UTF-8,EUC-JP' do
                   s1 = "\xe3\x83\xac\xe3\x83\x83\xe3\x83\x89\xe3\x83\x9e\xe3\x82\xa4\xe3\x83\xb3".force_encoding("UTF-8")
                   s2 = "\xa5\xec\xa5\xc3\xa5\xc9\xa5\xde\xa5\xa4\xa5\xf3".force_encoding("ASCII-8BIT")
                   s3 = s2.dup.force_encoding("UTF-8")
                   assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s2)
                   assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s3)
                 end
               end
               def test_to_utf8_by_setting_should_be_converted_all_latin1
                 with_settings :repositories_encodings => 'ISO-8859-1' do
                   s1 = "\xc3\x82\xc2\x80".force_encoding("UTF-8")
                   s2 = "\xC2\x80".force_encoding("ASCII-8BIT")
                   s3 = s2.dup.force_encoding("UTF-8")
                   assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s2)
                   assert_equal s1, Redmine::CodesetUtil.to_utf8_by_setting(s3)
                 end
               end
               def test_to_utf8_by_setting_blank_string
                 assert_equal "",  Redmine::CodesetUtil.to_utf8_by_setting("")
                 assert_nil Redmine::CodesetUtil.to_utf8_by_setting(nil)
               end
               def test_to_utf8_by_setting_returns_ascii_as_utf8
                 s1 = "ASCII".force_encoding("UTF-8")
                 s2 = s1.dup.force_encoding("ISO-8859-1")
                 str1 = Redmine::CodesetUtil.to_utf8_by_setting(s1)
                 str2 = Redmine::CodesetUtil.to_utf8_by_setting(s2)
                 assert_equal s1, str1
                 assert_equal s1, str2
                 assert_equal "UTF-8", str1.encoding.to_s
                 assert_equal "UTF-8", str2.encoding.to_s
               end
               def test_to_utf8_by_setting_invalid_utf8_sequences_should_be_stripped
                 with_settings :repositories_encodings => '' do
                   # s1 = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")
                   s1 = "Texte encod\xe9 en ISO-8859-1.".force_encoding("ASCII-8BIT")
                   str = Redmine::CodesetUtil.to_utf8_by_setting(s1)
                   assert str.valid_encoding?
                   assert_equal "UTF-8", str.encoding.to_s
                   assert_equal "Texte encod? en ISO-8859-1.", str
                 end
               end
               def test_to_utf8_by_setting_invalid_utf8_sequences_should_be_stripped_ja_jis
                 with_settings :repositories_encodings => 'ISO-2022-JP' do
                   s1 = "test\xb5\xfetest\xb5\xfe".force_encoding("ASCII-8BIT")
                   str = Redmine::CodesetUtil.to_utf8_by_setting(s1)
                   assert str.valid_encoding?
                   assert_equal "UTF-8", str.encoding.to_s
                   assert_equal "test??test??", str
                 end
               end
               test "#replace_invalid_utf8 should replace invalid utf8" do
                 s1 = "\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xE3\x81\xFF".force_encoding("UTF-8")
                 s2 = Redmine::CodesetUtil.replace_invalid_utf8(s1)
                 assert s2.valid_encoding?
                 assert_equal "UTF-8", s2.encoding.to_s
-                assert_equal "??????", s2
+                assert_equal "\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1??".force_encoding("UTF-8"), s2
               end
               test "#to_utf8 should replace invalid non utf8" do
                 s1 = "\xa4\xb3\xa4\xf3\xa4\xcb\xa4\xc1\xa4".force_encoding("EUC-JP")
                 s2 = Redmine::CodesetUtil.to_utf8(s1, "EUC-JP")
                 assert s2.valid_encoding?
                 assert_equal "UTF-8", s2.encoding.to_s
                 assert_equal "\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1?".force_encoding("UTF-8"), s2
               end
             end