CsTokenizerTest xref
1 /**
2 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3 */
4
5 package net.sourceforge.pmd.cpd;
6
7 import static org.junit.Assert.assertEquals;
8
9 import java.util.List;
10
11 import org.junit.Before;
12 import org.junit.Test;
13
14 public class CsTokenizerTest {
15
16 private CsTokenizer tokenizer = new CsTokenizer();
17
18 private Tokens tokens;
19
20 @Before
21 public void init() {
22 tokens = new Tokens();
23 TokenEntry.clearImages();
24 }
25
26 @Test
27 public void testSimpleClass() {
28 tokenizer.tokenize(toSourceCode("class Foo {}"), tokens);
29 assertEquals(5, tokens.size());
30 }
31
32 @Test
33 public void testSimpleClassDuplicatedTokens() {
34 tokenizer.tokenize(toSourceCode("class Foo { class Foo { } }"), tokens);
35 assertEquals(9, tokens.size());
36 List<TokenEntry> tokenList = tokens.getTokens();
37 assertEquals(tokenList.get(0).getIdentifier(), tokenList.get(3).getIdentifier());
38 assertEquals(tokenList.get(1).getIdentifier(), tokenList.get(4).getIdentifier());
39 assertEquals(tokenList.get(2).getIdentifier(), tokenList.get(5).getIdentifier());
40 assertEquals(tokenList.get(6).getIdentifier(), tokenList.get(7).getIdentifier());
41 }
42
43 @Test
44 public void testSimpleClassMethodMultipleLines() {
45 tokenizer.tokenize(toSourceCode(
46 "class Foo {\n"
47 + " public String foo(int a) {\n"
48 + " int i = a;\n"
49 + " return \"x\" + a;\n"
50 + " }\n"
51 + "}"), tokens);
52 assertEquals(22, tokens.size());
53 List<TokenEntry> tokenList = tokens.getTokens();
54 assertEquals(1, tokenList.get(0).getBeginLine());
55 assertEquals(2, tokenList.get(4).getBeginLine());
56 assertEquals(3, tokenList.get(11).getBeginLine());
57 }
58
59 @Test
60 public void testStrings() {
61 tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\n\";"), tokens);
62 assertEquals(5, tokens.size());
63 }
64
65 @Test
66 public void testOpenString() {
67 tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\"), tokens);
68 assertEquals(5, tokens.size());
69 }
70
71
72 @Test
73 public void testCommentsIgnored1() {
74 tokenizer.tokenize(toSourceCode("class Foo { /* class * ** X */ }"), tokens);
75 assertEquals(5, tokens.size());
76 }
77
78 @Test
79 public void testCommentsIgnored2() {
80 tokenizer.tokenize(toSourceCode("class Foo { // class X /* aaa */ \n }"), tokens);
81 assertEquals(5, tokens.size());
82 }
83
84 @Test
85 public void testCommentsIgnored3() {
86 tokenizer.tokenize(toSourceCode("class Foo { /// class X /* aaa */ \n }"), tokens);
87 assertEquals(5, tokens.size());
88 }
89
90 @Test
91 public void testMoreTokens() {
92 tokenizer.tokenize(toSourceCode(
93 "class Foo {\n"
94 + " void bar() {\n"
95 + " int a = 1 >> 2; \n"
96 + " a += 1; \n"
97 + " a++; \n"
98 + " a /= 3e2; \n"
99 + " float f = -3.1; \n"
100 + " f *= 2; \n"
101 + " bool b = ! (f == 2.0 || f >= 1.0 && f <= 2.0) \n"
102 + " }\n"
103 + "}"
104 ), tokens);
105 assertEquals(50, tokens.size());
106 }
107
108 private SourceCode toSourceCode(String source) {
109 return new SourceCode(new SourceCode.StringCodeLoader(source));
110 }
111 }