1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
|
from . import rfc
from . import language
from functools import reduce, wraps
from sys import getrecursionlimit, setrecursionlimit
try:
from cStringIO import StringIO
except:
from io import StringIO
# Same as rfc.parse_line(), but ensure it always returns a string with the
# same length as the input string.
def safe_parse_line(lang, state, **kwargs):
old_line = state.line
new_line, new_state = rfc.parse_line(lang, state, **kwargs)
assert len(new_line) == len(old_line)
return new_line, new_state
def make_state(line='', multi_end_stack=None, in_literal=None):
return rfc.State(line, multi_end_stack, in_literal)
def test_state():
"""
Verify constructor doesn't return a global default value.
"""
rfc.State().multi_end_stack.append('doh!')
assert(len(rfc.State().multi_end_stack) == 0)
def c_line(s):
"""
Given a string, return only the C comments, and in the column same position.
"""
line, state = safe_parse_line(language.c, make_state(s))
assert state.multi_end_stack == []
return line
def c_code(s):
"""
Given a string, return only the C code, and in the column same position.
"""
line, state = safe_parse_line(language.c, make_state(s), code_only=True)
assert state.multi_end_stack == []
return line
def py_line(s):
"""
Given a string, return only the Python comments, and in the column same position.
"""
line, state = safe_parse_line(language.python, make_state(s))
assert state.multi_end_stack == []
return line
def java_line(s):
"""
Given a string, return only the Java comments, and in the column same position.
Unlike c_line, java_line supports nested comments.
"""
line, state = safe_parse_line(language.java, make_state(s))
assert state.multi_end_stack == []
return line
def parse_code(lang, line):
return rfc.parse_code(lang, make_state(line))
def test_parse_code():
c = language.c
assert parse_code(c, '') == ('', make_state(''))
assert parse_code(c, '\n') == ('\n', make_state(''))
assert parse_code(c, 'foo // bar') == ('foo ', make_state('// bar'))
assert parse_code(c, 'foo /* bar') == ('foo ', make_state('/* bar')) # Ensure '*' is escapted.
assert parse_code(c, '/**/ foo') == ('', make_state('/**/ foo')) # Only return code /before/ comments.
# Multi-line Python strings are treated as comments.
assert parse_code(language.python, '""" bar """') == ('', make_state('""" bar """'))
def line_comment(lang, line, keep_tokens=True):
cmt, state = rfc.parse_line_comment(lang, make_state(line), keep_tokens)
return (cmt, state.line)
def test_parse_line_comment():
c = language.c
assert line_comment(c, '') == ('', '')
assert line_comment(c, '\n') == ('', '\n')
assert line_comment(c, '//\n') == ('//\n', '')
assert line_comment(c, '//\n', False) == (' \n', '')
def test_index_of_first_found():
assert rfc.index_of_first_found('', []) == 0 # Nothing to find.
assert rfc.index_of_first_found('', ['a']) == -1 # Nothing found.
assert rfc.index_of_first_found('ab', ['b']) == 1 # 'b' found.
assert rfc.index_of_first_found('abc', ['b', 'c']) == 1 # 'b' found first.
assert rfc.index_of_first_found('acb', ['b', 'c']) == 1 # 'c' found first.
def test_clear_line():
assert rfc.clear_line('') == ''
assert rfc.clear_line('abc') == ' '
assert rfc.clear_line('abc\n') == ' \n' # Preserve newline.
assert rfc.clear_line('abc\r\n') == ' \r\n' # Preserve multibyte newline.
def test_get_linesep():
assert rfc.get_linesep('') == ''
assert rfc.get_linesep('foo\n') == '\n'
assert rfc.get_linesep('foo\r\n') == '\r\n'
assert rfc.get_linesep('foo\r\nbar\n') == '\n'
def multiline_comment(lang, line, keep_tokens=True):
return rfc.parse_multiline_comment(lang, make_state(line), keep_tokens)
def test_parse_multiline_comment():
c = language.c
assert multiline_comment(c, '/**/\n') == ('/**/', make_state('\n'))
assert multiline_comment(c, '/**/\n', False) == (' ', make_state('\n'))
def declarations(lang, line, keep_tokens=True):
return rfc.parse_declarations(lang, make_state(line), keep_tokens=keep_tokens)
def test_parse_declarations():
c = language.c
assert declarations(c, '\n') == ('\n', make_state())
assert declarations(c, '/**/\n') == ('/**/\n', make_state())
def test_parse_line():
assert c_line('') == ''
assert c_line('no comments') == ' '
assert c_line('/**/') == '/**/'
assert c_line('/* a */') == '/* a */'
assert c_line('// a') == '// a'
assert py_line('""" a """') == '""" a """'
assert py_line("''' a '''") == "''' a '''"
# Preserve newline
assert c_line('/* a */\n') == '/* a */\n'
# Ensure column position is not modified.
assert c_line('abc /* a */') == ' /* a */'
assert c_line('abc // a') == ' // a'
# Test comments in comments
assert c_line('// /*abc*/') == '// /*abc*/'
assert c_line('/* a */ // a') == '/* a */ // a'
assert java_line('/* /**/ */') == '/* /**/ */'
assert java_line('/*/**/*/') == '/*/**/*/'
assert c_line('/*/**/*/') == '/*/**/ '
assert c_line('/* // */') == '/* // */'
assert py_line('"""# a"""') == '"""# a"""'
# Test strings with strings
assert c_line('"\\\"foo\\\""') == ' '
assert c_line('"foo') == ' '
# Test string literals with comments
assert c_line('"/*"') == ' '
assert c_line("'/*'") == ' '
# Test c_code
assert c_code('') == ''
assert c_code('no comments') == 'no comments'
assert c_code('/**/') == ' '
assert c_code('/* a */\n') == ' \n'
assert c_code('/* a */ abc') == ' abc'
assert c_code('abc /**/') == 'abc '
def string_literal(quote, line):
lit, state = rfc.parse_string_literal(quote, make_state(line))
return (lit, state.line)
def test_parse_string_literal():
assert string_literal('"', 'abc') == ('', 'abc')
assert string_literal('"', '"a"') == ('"a"', '')
assert string_literal("'", "'a'") == ("'a'", '')
assert string_literal("'", "'a'b") == ("'a'", 'b')
# String without an end quote.
assert string_literal('"', '"a') == ('"a', '')
def parse_line(lang, line, multi_end_stack=None, code_only=False):
return safe_parse_line(lang, make_state(line, multi_end_stack), code_only=code_only)
def test_incomplete_multiline():
c = language.c
assert parse_line(c, '/* a\n') == ('/* a\n', make_state('', ['*/']))
def test_incomplete_string_literal():
c = language.c
assert parse_line(c, '" a \\\n', code_only=True) == ('" a \\\n', make_state('', None, '"'))
def test_previous_state_maintained():
c = language.c
assert parse_line(c, '', ['foo']) == ('', make_state('', ['foo']))
assert parse_line(c, '/**/', ['foo']) == ('/**/', make_state('', ['foo']))
assert parse_line(c, '/*', ['foo']) == ('/*', make_state('', ['foo']))
j = language.java
assert parse_line(j, '/*', ['foo']) == ('/*', make_state('', ['foo', '*/']))
def test_code_of_resumed_multiline_comment():
c = language.c
assert parse_line(c, 'a', ['*/'], True) == (' ', make_state('', ['*/']))
assert parse_line(c, 'a */', ['*/'], True) == (' ', make_state('', []))
def parse_lit_line(lang, line, code_only=False):
return safe_parse_line(lang, make_state(line, None, lang.string_literal_start), code_only=code_only)
def test_previous_state_maintained_literal():
c = language.c
assert parse_lit_line(c, '') == ('', make_state('', in_literal='"'))
assert parse_lit_line(c, '"') == (' ', make_state(''))
def test_code_of_resumed_multiline_literal():
c = language.c
assert parse_lit_line(c, '', True) == ('', make_state('', in_literal='"'))
assert parse_lit_line(c, '"', True) == ('"', make_state(''))
def c_comments(s, keep_tokens=True):
return list(rfc.parse_file(language.c, StringIO(s), keep_tokens=keep_tokens))
def test_parse_file():
assert c_comments('/* hello */ world\n') == ['/* hello */ \n']
assert c_comments('/* hello */ world\n', False) == [' hello \n']
def test_parse_comments_via_reduce():
def f(st, x):
st.line = x
_, st = safe_parse_line(language.c, st)
return st
assert reduce(f, ['/*a', 'b*/'], make_state()) == make_state()
assert reduce(f, ['/*a', 'b'], make_state()) == make_state('', ['*/'])
|
