1 /***
2 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3 */
4 package net.sourceforge.pmd.cpd;
5
6 import net.sourceforge.pmd.cpd.cppast.CPPParser;
7 import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager;
8 import net.sourceforge.pmd.cpd.cppast.Token;
9 import net.sourceforge.pmd.cpd.cppast.TokenMgrError;
10
11 import java.io.IOException;
12 import java.io.LineNumberReader;
13 import java.io.Reader;
14 import java.io.StringReader;
15 import java.util.ArrayList;
16 import java.util.List;
17
18 public class CPPTokenizer implements Tokenizer {
19 protected String EOL = System.getProperty("line.separator", "\n");
20
21 private static boolean initted;
22
23 public void tokenize(SourceCode sourceCode, Tokens tokenEntries, Reader input) throws IOException {
24 try {
25 // first get a snapshot of the code
26 List lines = new ArrayList();
27 StringBuffer sb = new StringBuffer();
28 LineNumberReader r = new LineNumberReader(input);
29 String currentLine;
30 while ((currentLine = r.readLine()) != null) {
31 lines.add(currentLine);
32 sb.append(currentLine);
33 sb.append(EOL);
34 }
35 sourceCode.setCode(lines);
36
37 // now tokenize it
38 if (!initted) {
39 new CPPParser(new StringReader(sb.toString()));
40 initted = true;
41 }
42 CPPParser.ReInit(new StringReader(sb.toString()));
43 Token currToken = CPPParserTokenManager.getNextToken();
44 int count = 0;
45 while (currToken.image != "") {
46 count++;
47 tokenEntries.add(new TokenEntry(currToken.image, count, sourceCode.getFileName(), currToken.beginLine));
48 currToken = CPPParserTokenManager.getNextToken();
49 }
50 tokenEntries.add(TokenEntry.EOF);
51 System.out.println("Added " + sourceCode.getFileName());
52 } catch (TokenMgrError err) {
53 System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error");
54 List emptyCode = new ArrayList();
55 emptyCode.add("");
56 sourceCode.setCode(emptyCode);
57 tokenEntries.add(TokenEntry.EOF);
58 }
59 }
60 }
This page was automatically generated by Maven