fan out of changes to make more generic with grammar start rule

Author: parrt

first-grammar-formatting-result-sight-unseen.txt

@@ -0,0 +1,347 @@
+/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/bin/java -Didea.launcher.port=7540 "-Didea.launcher.bin.path=/Applications/IntelliJ IDEA.app/Contents/bin" -Dfile.encoding=UTF-8 -classpath "/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/lib/ant-javafx.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/lib/dt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/lib/javafx-mx.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/lib/jconsole.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/lib/packager.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/lib/sa-jdi.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/lib/tools.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/charsets.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/deploy.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/javaws.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/jce.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/jfr.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/jfxswt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/jsse.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/management-agent.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/plugin.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/resources.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/rt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/cldrdata.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/dnsns.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/jaccess.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/jfxrt.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/localedata.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/nashorn.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/sunec.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/sunjce_provider.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/sunpkcs11.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_60.jdk/Contents/Home/jre/lib/ext/zipfs.jar:/Users/parrt/antlr/code/codebuff/java/target/classes:/Users/parrt/.m2/repository/org/antlr/antlr4-runtime/4.5.3/antlr4-runtime-4.5.3.jar:/Users/parrt/.m2/repository/org/apache/commons/commons-lang3/3.4/commons-lang3-3.4.jar:/Applications/IntelliJ IDEA.app/Contents/lib/idea_rt.jar" com.intellij.rt.execution.application.AppMain org.antlr.codebuff.Tool -antlr grammars grammars/org/antlr/codebuff/ANTLRv4Parser.g4
+3 files
+3233 feature vectors
+
+
+Incorrect_WS / All_WS: 119 / 487 = 24.4%
+misclassified: 0
+Diff is 0.17840805649757385
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2013 Terence Parr
+ *  Copyright (c) 2013 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** A grammar for ANTLR v4 written in ANTLR v4.
+*/
+parser grammar ANTLRv4Parser;
+
+options {
+    tokenVocab=ANTLRv4Lexer;
+}
+
+
+grammarSpec
+    :   DOC_COMMENT? grammarType id SEMI prequelConstruct* rules modeSpec* EOF
+;
+
+grammarType
+    :   (  LEXER GRAMMAR| PARSER GRAMMAR|GRAMMAR)
+    ;
+
+prequelConstruct
+    :   optionsSpec
+    |   delegateGrammars
+    |   tokensSpec
+    |   channelsSpec
+    |   action
+;
+
+optionsSpec
+    :   OPTIONS (option SEMI)* RBRACE
+;
+
+option
+    :   id ASSIGN optionValue
+;
+
+optionValue
+    :   id (DOT id)*
+    |   STRING_LITERAL
+    |   ACTION
+    |   INT
+;
+
+delegateGrammars
+    :   IMPORT delegateGrammar (COMMA delegateGrammar)* SEMI
+;
+
+delegateGrammar
+    :   id ASSIGN id
+    |   id
+;
+
+tokensSpec
+    :   TOKENS idList RBRACE
+;
+
+channelsSpec
+    :   CHANNELS idList? RBRACE
+;
+
+idList
+    :   id (COMMA id)* COMMA?
+;
+
+/** Match stuff like @parser::members {int i;} */
+    action
+    :   AT (actionScopeName COLONCOLON)?
+           id ACTION
+;
+
+/** Sometimes the scope names will collide with keywords; allow them as
+ *  ids for action scopes.
+ */
+    actionScopeName
+    :   id
+    |   LEXER
+    |   PARSER
+;
+
+modeSpec
+    :   MODE id SEMI lexerRule*
+;
+
+rules
+    :   ruleSpec*
+;
+
+ruleSpec
+    :   parserRuleSpec
+    |   lexerRule
+;
+
+parserRuleSpec
+    :   DOC_COMMENT? ruleModifiers? RULE_REF ARG_ACTION? ruleReturns? throwsSpec? localsSpec? rulePrequel* COLON ruleBlock SEMI exceptionGroup
+;
+
+exceptionGroup
+    :   exceptionHandler* finallyClause?
+;
+
+exceptionHandler
+    :   CATCH ARG_ACTION ACTION
+;
+
+finallyClause
+    :   FINALLY ACTION
+;
+
+rulePrequel
+    :   optionsSpec
+    |   ruleAction
+;
+
+ruleReturns
+    :   RETURNS ARG_ACTION
+;
+
+throwsSpec
+    :   THROWS id (COMMA id)*
+;
+
+localsSpec
+    :   LOCALS ARG_ACTION
+;
+
+/** Match stuff like @init {int i;} */
+    ruleAction
+    :   AT id ACTION
+;
+
+ruleModifiers
+    :   ruleModifier+
+;
+
+ruleModifier
+    :   PUBLIC
+    |   PRIVATE
+    |   PROTECTED
+    |   FRAGMENT
+;
+
+ruleBlock
+    :   ruleAltList
+;
+
+ruleAltList
+    :   labeledAlt (OR labeledAlt)*
+;
+
+labeledAlt
+    :   alternative (POUND id)?
+;
+
+lexerRule
+    :   DOC_COMMENT? FRAGMENT? TOKEN_REF COLON lexerRuleBlock SEMI
+;
+
+lexerRuleBlock
+    :   lexerAltList
+;
+
+lexerAltList
+    :   lexerAlt (OR lexerAlt)*
+;
+
+lexerAlt
+    :   lexerElements lexerCommands?
+    |
+;
+
+lexerElements
+    :   lexerElement+
+;
+
+lexerElement
+    :   labeledLexerElement ebnfSuffix?
+    |   lexerAtom ebnfSuffix?
+    |   lexerBlock ebnfSuffix?
+    |   ACTION QUESTION?
+;
+
+labeledLexerElement
+    :   id (ASSIGN|PLUS_ASSIGN)
+        (lexerAtom| block)
+    ;
+
+lexerBlock
+    :   LPAREN lexerAltList RPAREN
+;
+
+lexerCommands
+    :   RARROW lexerCommand (COMMA lexerCommand)*
+;
+
+lexerCommand
+    :   lexerCommandName LPAREN lexerCommandExpr RPAREN
+    |   lexerCommandName
+;
+
+lexerCommandName
+    :   id
+    |   MODE
+;
+
+lexerCommandExpr
+    :   id
+    |   INT
+;
+
+altList
+    :   alternative (OR alternative)*
+;
+
+alternative
+    :   elementOptions? element*
+;
+
+element
+    :   labeledElement (ebnfSuffix
+                       |
+                       )
+    |   atom (ebnfSuffix
+             |
+             )
+    |   ebnf
+    |   ACTION QUESTION?
+;
+
+labeledElement
+    :   id (ASSIGN|PLUS_ASSIGN)
+        (atom| block)
+    ;
+
+ebnf
+    :   block blockSuffix?
+;
+
+blockSuffix
+    :   ebnfSuffix
+;
+
+ebnfSuffix
+    :   QUESTION QUESTION?
+    |   STAR QUESTION?
+    |   PLUS QUESTION?
+;
+
+lexerAtom
+    :   range
+    |   terminal
+    |   RULE_REF
+    |   notSet
+    |   LEXER_CHAR_SET
+    |   DOT elementOptions?
+;
+
+atom
+    :   range
+    |   terminal
+    |   ruleref
+    |   notSet
+    |   DOT elementOptions?
+;
+
+notSet
+    :   NOT setElement
+    |   NOT blockSet
+;
+
+blockSet
+    :   LPAREN setElement (OR setElement)* RPAREN
+;
+
+setElement
+    :   TOKEN_REF elementOptions?
+    |   STRING_LITERAL elementOptions?
+    |   range
+    |   LEXER_CHAR_SET
+;
+
+block
+    :   LPAREN (optionsSpec? ruleAction* COLON)?
+               altList RPAREN
+;
+
+ruleref
+    :   RULE_REF ARG_ACTION? elementOptions?
+;
+
+range
+    :   STRING_LITERAL RANGE STRING_LITERAL
+;
+
+terminal
+    :   TOKEN_REF elementOptions?
+    |   STRING_LITERAL elementOptions?
+;
+
+elementOptions
+    :   LT elementOption (COMMA elementOption)* GT
+;
+
+elementOption
+    :   id
+    |   id ASSIGN (id|STRING_LITERAL)
+    ;
+
+id
+    :   RULE_REF
+    |   TOKEN_REF
+;
+

java/grammars/org/antlr/codebuff/ANTLRv4Lexer.g4

@@ -75,17 +75,8 @@ tokens {
 
 	@Override
 	public Token emit() {
-		if (_type == /*ID*/ 999999) { // ################### just testing
-			String firstChar = _input.getText(Interval.of(_tokenStartCharIndex, _tokenStartCharIndex));
-			if (Character.isUpperCase(firstChar.charAt(0))) {
-				_type = TOKEN_REF;
-			} else {
-				_type = RULE_REF;
-			}
-
-			if (_currentRuleType == Token.INVALID_TYPE) { // if outside of rule def
-				_currentRuleType = _type;                 // set to inside lexer or parser rule
-			}
+		if (_type == TOKEN_REF || _type == RULE_REF ) {
+            _currentRuleType = _type;
 		}
 		else if (_type == SEMI) {                  // exit rule def
 			_currentRuleType = Token.INVALID_TYPE;

java/pom.xml

@@ -51,6 +51,7 @@
 
 	<build>
 		<sourceDirectory>src</sourceDirectory>
+		<testSourceDirectory>test</testSourceDirectory>
 		<plugins>
 			<plugin>
 				<groupId>org.antlr</groupId>

java/src/org/antlr/codebuff/CollectFeatures.java

@@ -479,7 +479,7 @@ public static int[] getNodeFeatures(Map<Token, TerminalNode> tokenToNodeMap,
 		ParserRuleContext ancestorParent = null;
 		ParserRuleContext ancestorParent2 = null;
 		if ( earliestLeftAncestor==null ) { // just use regular parent then
-			ancestorParent = getParent(node);
+			ancestorParent = (ParserRuleContext)node.getParent();
 			ancestorParent2 = ancestorParent.getParent(); // get immediate parent for context
 		}
 		else {
@@ -544,7 +544,7 @@ public static TerminalNode getMatchingLeftSymbol(InputDocument doc,
 		int curTokensParentRuleIndex = parent.getRuleIndex();
 		Token curToken = node.getSymbol();
 		if (ruleToPairsBag != null) {
-			String ruleName = JavaParser.ruleNames[curTokensParentRuleIndex];
+			String ruleName = doc.parser.getRuleNames()[curTokensParentRuleIndex];
 			List<Pair<Integer, Integer>> pairs = ruleToPairsBag.get(ruleName);
 			if ( pairs!=null ) {
 				// Find appropriate pair given current token