View Javadoc

1   /***
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3    */
4   package net.sourceforge.pmd.cpd;
5   
6   import net.sourceforge.pmd.cpd.cppast.CPPParser;
7   import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager;
8   import net.sourceforge.pmd.cpd.cppast.Token;
9   import net.sourceforge.pmd.cpd.cppast.TokenMgrError;
10  
11  import java.io.StringReader;
12  import java.util.ArrayList;
13  import java.util.List;
14  
15  public class CPPTokenizer implements Tokenizer {
16      protected String EOL = System.getProperty("line.separator", "\n");
17  
18      private static boolean initted;
19  
20      public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
21          StringBuffer sb = sourceCode.getCodeBuffer();
22          try {
23              if (!initted) {
24                  new CPPParser(new StringReader(sb.toString()));
25                  initted = true;
26              }
27              CPPParser.ReInit(new StringReader(sb.toString()));
28              Token currToken = CPPParserTokenManager.getNextToken();
29              while (currToken.image.length() > 0) {
30                  tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine));
31                  currToken = CPPParserTokenManager.getNextToken();
32              }
33              tokenEntries.add(TokenEntry.getEOF());
34              System.out.println("Added " + sourceCode.getFileName());
35          } catch (TokenMgrError err) {
36              System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error");
37              List emptyCode = new ArrayList();
38              emptyCode.add("");
39              tokenEntries.add(TokenEntry.getEOF());
40          }
41      }
42  }