View Javadoc
1 /*** 2 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html 3 */ 4 package net.sourceforge.pmd.cpd; 5 6 import net.sourceforge.pmd.cpd.cppast.CPPParser; 7 import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager; 8 import net.sourceforge.pmd.cpd.cppast.Token; 9 import net.sourceforge.pmd.cpd.cppast.TokenMgrError; 10 11 import java.io.IOException; 12 import java.io.LineNumberReader; 13 import java.io.Reader; 14 import java.io.StringReader; 15 import java.util.ArrayList; 16 import java.util.List; 17 18 public class CPPTokenizer implements Tokenizer { 19 protected String EOL = System.getProperty("line.separator", "\n"); 20 21 private static boolean initted; 22 23 public void tokenize(SourceCode sourceCode, Tokens tokenEntries, Reader input) throws IOException { 24 try { 25 // first get a snapshot of the code 26 List lines = new ArrayList(); 27 StringBuffer sb = new StringBuffer(); 28 LineNumberReader r = new LineNumberReader(input); 29 String currentLine; 30 while ((currentLine = r.readLine()) != null) { 31 lines.add(currentLine); 32 sb.append(currentLine); 33 sb.append(EOL); 34 } 35 sourceCode.setCode(lines); 36 37 // now tokenize it 38 if (!initted) { 39 new CPPParser(new StringReader(sb.toString())); 40 initted = true; 41 } 42 CPPParser.ReInit(new StringReader(sb.toString())); 43 Token currToken = CPPParserTokenManager.getNextToken(); 44 int count = 0; 45 while (currToken.image != "") { 46 count++; 47 tokenEntries.add(new TokenEntry(currToken.image, count, sourceCode.getFileName(), currToken.beginLine)); 48 currToken = CPPParserTokenManager.getNextToken(); 49 } 50 tokenEntries.add(TokenEntry.EOF); 51 System.out.println("Added " + sourceCode.getFileName()); 52 } catch (TokenMgrError err) { 53 System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error"); 54 List emptyCode = new ArrayList(); 55 emptyCode.add(""); 56 sourceCode.setCode(emptyCode); 57 tokenEntries.add(TokenEntry.EOF); 58 } 59 } 60 }

This page was automatically generated by Maven