Generate dynamic lexer from antlr grammar

Generate dynamic lexer from antlr grammar and try to parse something, because it is a lexer only, so it returns tokens not grammar tree.

package hk.quantr.antlrparserofparser;

import java.io.FileInputStream;
import java.io.IOException;
import java.util.logging.Level;
import org.antlr.v4.Tool;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.LexerInterpreter;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.tool.Grammar;
import org.antlr.v4.tool.ast.GrammarRootAST;
import org.apache.commons.io.IOUtils;
import org.apache.log4j.Logger;
import org.junit.Test;

/**
 *
 * @author Peter <peter@quantr.hk>
 */
public class Test1 {

	Logger logger = Logger.getLogger(Test1.class);

	@Test
	public void test1() {
		try {
			Tool tool = new Tool();
//			String g4 = IOUtils.toString(getClass().getResourceAsStream("Assembler.g4"), "utf-8");
			String g4 = IOUtils.toString(new FileInputStream("/Users/peter/workspace/Assembler/src/main/java/hk/quantr/assembler/antlr/Assembler.g4"), "utf-8");

			GrammarRootAST ast = tool.parseGrammarFromString(g4);
			if (ast.grammarType == ANTLRParser.COMBINED) {
				Grammar grammar = tool.createGrammar(ast);
				tool.process(grammar, false);

				LexerInterpreter lexer = grammar.createLexerInterpreter(new ANTLRInputStream("(bits 64){aaa ;fuck}"));
				for (Token token : lexer.getAllTokens()) {
					logger.info(token + " > " + lexer.getVocabulary().getSymbolicName(token.getType()));
				}
			}
		} catch (IOException ex) {
			java.util.logging.Logger.getLogger(Test1.class.getName()).log(Level.SEVERE, null, ex);
		}
	}
}