View Javadoc
1 /*** 2 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html 3 */ 4 package net.sourceforge.pmd.cpd; 5 6 import net.sourceforge.pmd.PMD; 7 import net.sourceforge.pmd.ast.JavaCharStream; 8 import net.sourceforge.pmd.ast.JavaParserTokenManager; 9 import net.sourceforge.pmd.ast.Token; 10 11 import java.io.IOException; 12 import java.io.LineNumberReader; 13 import java.io.Reader; 14 import java.io.StringReader; 15 import java.util.ArrayList; 16 import java.util.List; 17 18 public class JavaTokenizer implements Tokenizer { 19 20 public void tokenize(SourceCode tokens, Tokens tokenEntries, Reader input) throws IOException { 21 // first get a snapshot of the code 22 List lines = new ArrayList(); 23 StringBuffer sb = new StringBuffer(); 24 LineNumberReader r = new LineNumberReader(input); 25 String currentLine; 26 while ((currentLine = r.readLine()) != null) { 27 lines.add(currentLine); 28 sb.append(currentLine); 29 sb.append(PMD.EOL); 30 } 31 tokens.setCode(lines); 32 33 // now tokenize it 34 /* 35 I'm doing a sort of State pattern thing here where 36 this goes into "discarding" mode when it hits an import or package 37 keyword and goes back into "accumulate mode when it hits a semicolon. 38 This could probably be turned into some objects. 39 */ 40 JavaCharStream javaStream = new JavaCharStream(new StringReader(sb.toString())); 41 JavaParserTokenManager tokenMgr = new JavaParserTokenManager(javaStream); 42 Token currToken = tokenMgr.getNextToken(); 43 boolean discarding = false; 44 int count = 0; 45 while (currToken.image != "") { 46 if (currToken.image.equals("import") || currToken.image.equals("package")) { 47 discarding = true; 48 currToken = tokenMgr.getNextToken(); 49 continue; 50 } 51 52 if (discarding && currToken.image.equals(";")) { 53 discarding = false; 54 } 55 56 if (discarding) { 57 currToken = tokenMgr.getNextToken(); 58 continue; 59 } 60 61 if (!currToken.image.equals(";")) { 62 count++; 63 tokenEntries.add(new TokenEntry(currToken.image, count, tokens.getFileName(), currToken.beginLine)); 64 } 65 66 currToken = tokenMgr.getNextToken(); 67 } 68 tokenEntries.add(TokenEntry.EOF); 69 } 70 }

This page was automatically generated by Maven