jro_apps/redmine Commit - r1462:6ebdf848a583

1

module CodeRay

1

module CodeRay

2

module Scanners

2

module Scanners

3

4

class C < Scanner

4

class C < Scanner

5

6

register_for :c

6

register_for :c

7

8

include Streamable

8

include Streamable

9

10

RESERVED_WORDS = [

10

RESERVED_WORDS = [

11

'asm', 'break', 'case', 'continue', 'default', 'do', 'else',

11

'asm', 'break', 'case', 'continue', 'default', 'do', 'else',

12

'for', 'goto', 'if', 'return', 'switch', 'while',

12

'for', 'goto', 'if', 'return', 'switch', 'while',

13

'struct', 'union', 'enum', 'typedef',

13

'struct', 'union', 'enum', 'typedef',

14

'static', 'register', 'auto', 'extern',

14

'static', 'register', 'auto', 'extern',

15

'sizeof',

15

'sizeof',

16

'volatile', 'const', # C89

16

'volatile', 'const', # C89

17

'inline', 'restrict', # C99

17

'inline', 'restrict', # C99

18

]

18

]

19

20

PREDEFINED_TYPES = [

20

PREDEFINED_TYPES = [

21

'int', 'long', 'short', 'char', 'void',

21

'int', 'long', 'short', 'char', 'void',

22

'signed', 'unsigned', 'float', 'double',

22

'signed', 'unsigned', 'float', 'double',

23

'bool', 'complex', # C99

23

'bool', 'complex', # C99

24

]

24

]

25

26

PREDEFINED_CONSTANTS = [

26

PREDEFINED_CONSTANTS = [

27

'EOF', 'NULL',

27

'EOF', 'NULL',

28

'true', 'false', # C99

28

'true', 'false', # C99

29

]

29

]

30

31

IDENT_KIND = WordList.new(:ident).

31

IDENT_KIND = WordList.new(:ident).

32

add(RESERVED_WORDS, :reserved).

32

add(RESERVED_WORDS, :reserved).

33

add(PREDEFINED_TYPES, :pre_type).

33

add(PREDEFINED_TYPES, :pre_type).

34

add(PREDEFINED_CONSTANTS, :pre_constant)

34

add(PREDEFINED_CONSTANTS, :pre_constant)

35

36

ESCAPE = / [rbfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x

36

ESCAPE = / [rbfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x

37

UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x

37

UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x

38

39

def scan_tokens tokens, options

39

def scan_tokens tokens, options

40

41

state = :initial

41

state = :initial

42

43

until eos?

43

until eos?

44

45

kind = nil

45

kind = nil

46

match = nil

46

match = nil

47

48

case state

48

case state

49

50

when :initial

50

when :initial

51

52

if scan(/ \s+ | \\\n /x)

52

if scan(/ \s+ | \\\n /x)

53

kind = :space

53

kind = :space

54

55

elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)

55

elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)

56

kind = :comment

56

kind = :comment

57

58

elsif match = scan(/ \# \s* if \s* 0 /x)

58

elsif match = scan(/ \# \s* if \s* 0 /x)

59

match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?

59

match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?

60

kind = :comment

60

kind = :comment

61

62

elsif scan(/ [-+*\/=<>?:;,!&^|()\[\]{}~%]+ | \.(?!\d) /x)

62

elsif scan(/ [-+*\/=<>?:;,!&^|()\[\]{}~%]+ | \.(?!\d) /x)

63

kind = :operator

63

kind = :operator

64

65

elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)

65

elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)

66

kind = IDENT_KIND[match]

66

kind = IDENT_KIND[match]

67

if kind == :ident and check(/:(?!:)/)

67

if kind == :ident and check(/:(?!:)/)

68

match << scan(/:/)

68

match << scan(/:/)

69

kind = :label

69

kind = :label

70

end

70

end

71

72

elsif match = scan(/L?"/)

72

elsif match = scan(/L?"/)

73

tokens << [:open, :string]

73

tokens << [:open, :string]

74

if match[0] == ?L

74

if match[0] == ?L

75

tokens << ['L', :modifier]

75

tokens << ['L', :modifier]

76

match = '"'

76

match = '"'

77

end

77

end

78

state = :string

78

state = :string

79

kind = :delimiter

79

kind = :delimiter

80

81

elsif scan(/#\s*(\w*)/)

81

elsif scan(/#\s*(\w*)/)

82

kind = :preprocessor # FIXME multiline preprocs

82

kind = :preprocessor # FIXME multiline preprocs

83

state = :include_expected if self[1] == 'include'

83

state = :include_expected if self[1] == 'include'

84

85

elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)

85

elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)

86

kind = :char

86

kind = :char

87

88

elsif scan(/0[xX][0-9A-Fa-f]+/)

88

elsif scan(/0[xX][0-9A-Fa-f]+/)

89

kind = :hex

89

kind = :hex

90

91

elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)

91

elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)

92

kind = :oct

92

kind = :oct

93

94

elsif scan(/(?:\d+)(?![.eEfF])/)

94

elsif scan(/(?:\d+)(?![.eEfF])/)

95

kind = :integer

95

kind = :integer

96

97

elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)

97

elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)

98

kind = :float

98

kind = :float

99

100

else

100

else

101

getch

101

getch

102

kind = :error

102

kind = :error

103

104

end

104

end

105

106

when :string

106

when :string

107

if scan(/[^\\\n"]+/)

107

if scan(/[^\\\n"]+/)

108

kind = :content

108

kind = :content

109

elsif scan(/"/)

109

elsif scan(/"/)

110

tokens << ['"', :delimiter]

110

tokens << ['"', :delimiter]

111

tokens << [:close, :string]

111

tokens << [:close, :string]

112

state = :initial

112

state = :initial

113

elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)

114

elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)

115

kind = :char

115

kind = :char

116

elsif scan(/ \\ | $ /x)

116

elsif scan(/ \\ | $ /x)

117

tokens << [:close, :string]

117

tokens << [:close, :string]

118

kind = :error

118

kind = :error

119

state = :initial

119

state = :initial

120

else

120

else

121

raise_inspect "else case \" reached; %p not handled." % peek(1), tokens

121

raise_inspect "else case \" reached; %p not handled." % peek(1), tokens

122

end

122

end

123

124

when :include_expected

124

when :include_expected

125

if scan(/<[^>\n]+~~>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?~~/)

125

if scan(/[^\n]+/)

126

kind = :include

126

kind = :include

127

state = :initial

127

state = :initial

128

129

elsif match = scan(/\s+/)

129

elsif match = scan(/\s+/)

130

kind = :space

130

kind = :space

131

state = :initial if match.index ?\n

131

state = :initial if match.index ?\n

132

133

else

133

else

134

getch

134

getch

135

kind = :error

135

kind = :error

136

137

end

137

end

138

139

else

139

else

140

raise_inspect 'Unknown state', tokens

140

raise_inspect 'Unknown state', tokens

141

142

end

142

end

143

144

match ||= matched

144

match ||= matched

145

if $DEBUG and not kind

145

if $DEBUG and not kind

146

raise_inspect 'Error token %p in line %d' %

146

raise_inspect 'Error token %p in line %d' %

147

[[match, kind], line], tokens

147

[[match, kind], line], tokens

148

end

148

end

149

raise_inspect 'Empty token', tokens unless match

149

raise_inspect 'Empty token', tokens unless match

150

151

tokens << [match, kind]

151

tokens << [match, kind]

152

153

end

153

end

154

155

if state == :string

155

if state == :string

156

tokens << [:close, :string]

156

tokens << [:close, :string]

157

end

157

end

158

159

tokens

159

tokens

160

end

160

end

161

162

end

162

end

163

164

end

164

end

165

end

165

end

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages

             module CodeRay
             module Scanners
               class C < Scanner
                 register_for :c
                 include Streamable
                 RESERVED_WORDS = [
                   'asm', 'break', 'case', 'continue', 'default', 'do', 'else',
                   'for', 'goto', 'if', 'return', 'switch', 'while',
                   'struct', 'union', 'enum', 'typedef',
                   'static', 'register', 'auto', 'extern',
                   'sizeof',
                   'volatile', 'const',  # C89
                   'inline', 'restrict', # C99
                 ]
                 PREDEFINED_TYPES = [
                   'int', 'long', 'short', 'char', 'void',
                   'signed', 'unsigned', 'float', 'double',
                   'bool', 'complex',  # C99
                 ]
                 PREDEFINED_CONSTANTS = [
                   'EOF', 'NULL',
                   'true', 'false',  # C99
                 ]
                 IDENT_KIND = WordList.new(:ident).
                   add(RESERVED_WORDS, :reserved).
                   add(PREDEFINED_TYPES, :pre_type).
                   add(PREDEFINED_CONSTANTS, :pre_constant)
                 ESCAPE = / [rbfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
                 UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
                 def scan_tokens tokens, options
                   state = :initial
                   until eos?
                     kind = nil
                     match = nil
                     case state
                     when :initial
                       if scan(/ \s+ | \\\n /x)
                         kind = :space
                       elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
                         kind = :comment
                       elsif match = scan(/ \# \s* if \s* 0 /x)
                         match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
                         kind = :comment
                       elsif scan(/ [-+*\/=<>?:;,!&^|()\[\]{}~%]+ | \.(?!\d) /x)
                         kind = :operator
                       elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
                         kind = IDENT_KIND[match]
                         if kind == :ident and check(/:(?!:)/)
                           match << scan(/:/)
                           kind = :label
                         end
                       elsif match = scan(/L?"/)
                         tokens << [:open, :string]
                         if match[0] == ?L
                           tokens << ['L', :modifier]
                           match = '"'
                         end
                         state = :string
                         kind = :delimiter
                       elsif scan(/#\s*(\w*)/)
                         kind = :preprocessor  # FIXME multiline preprocs
                         state = :include_expected if self[1] == 'include'
                       elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
                         kind = :char
                       elsif scan(/0[xX][0-9A-Fa-f]+/)
                         kind = :hex
                       elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
                         kind = :oct
                       elsif scan(/(?:\d+)(?![.eEfF])/)
                         kind = :integer
                       elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
                         kind = :float
                       else
                         getch
                         kind = :error
                       end
                     when :string
                       if scan(/[^\\\n"]+/)
                         kind = :content
                       elsif scan(/"/)
                         tokens << ['"', :delimiter]
                         tokens << [:close, :string]
                         state = :initial
                         next
                       elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
                         kind = :char
                       elsif scan(/ \\ | $ /x)
                         tokens << [:close, :string]
                         kind = :error
                         state = :initial
                       else
                         raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
                       end
                     when :include_expected
-                      if scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
+                      if scan(/[^\n]+/)
                         kind = :include
                         state = :initial
                       elsif match = scan(/\s+/)
                         kind = :space
                         state = :initial if match.index ?\n
                       else
                         getch
                         kind = :error
                       end
                     else
                       raise_inspect 'Unknown state', tokens
                     end
                     match ||= matched
                     if $DEBUG and not kind
                       raise_inspect 'Error token %p in line %d' %
                         [[match, kind], line], tokens
                     end
                     raise_inspect 'Empty token', tokens unless match
                     tokens << [match, kind]
                   end
                   if state == :string
                     tokens << [:close, :string]
                   end
                   tokens
                 end
               end
             end
             end