1 /*** 2 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html 3 */ 4 package net.sourceforge.pmd.cpd; 5 6 import net.sourceforge.pmd.cpd.cppast.CPPParser; 7 import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager; 8 import net.sourceforge.pmd.cpd.cppast.Token; 9 import net.sourceforge.pmd.cpd.cppast.TokenMgrError; 10 11 import java.io.StringReader; 12 import java.util.ArrayList; 13 import java.util.List; 14 15 public class CPPTokenizer implements Tokenizer { 16 protected String EOL = System.getProperty("line.separator", "\n"); 17 18 private static boolean initted; 19 20 public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { 21 StringBuffer sb = sourceCode.getCodeBuffer(); 22 try { 23 if (!initted) { 24 new CPPParser(new StringReader(sb.toString())); 25 initted = true; 26 } 27 CPPParser.ReInit(new StringReader(sb.toString())); 28 Token currToken = CPPParserTokenManager.getNextToken(); 29 while (currToken.image.length() > 0) { 30 tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine)); 31 currToken = CPPParserTokenManager.getNextToken(); 32 } 33 tokenEntries.add(TokenEntry.getEOF()); 34 System.out.println("Added " + sourceCode.getFileName()); 35 } catch (TokenMgrError err) { 36 System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error"); 37 List emptyCode = new ArrayList(); 38 emptyCode.add(""); 39 tokenEntries.add(TokenEntry.getEOF()); 40 } 41 } 42 }