rename a var. update comment for Dbg, simplify the code.

Author: parrt

src/org/antlr/codebuff/Dbg.java

@@ -23,13 +23,13 @@
  * Testing:
  *
  * Dbg  -antlr     corpus/antlr4/training      grammars/org/antlr/codebuff/tsql.g4
- * Dbg  -leave-one-out -antlr     corpus/antlr4/training      corpus/antlr4/training/MASM.g4
- * Dbg  -leave-one-out -quorum     corpus/quorum/training      corpus/quorum/training/Containers/List.quorum
+ * Dbg  -antlr     corpus/antlr4/training      corpus/antlr4/training/MASM.g4
+ * Dbg  -quorum     corpus/quorum/training      corpus/quorum/training/Containers/List.quorum
  * Dbg  -sqlite    corpus/sqlclean/training      corpus/sqlclean/training/dmart_bits.sql
- * Dbg  -leave-one-out -tsql      corpus/sqlclean/training        corpus/sqlclean/training/dmart_bits_PSQLRPT24.sql
+ * Dbg  -tsql      corpus/sqlclean/training        corpus/sqlclean/training/dmart_bits_PSQLRPT24.sql
  * Dbg  -java      corpus/java/training/stringtemplate4     src/org/antlr/codebuff/Tool.java
- * Dbg  -leave-one-out -java      corpus/java/training/stringtemplate4     corpus/java/training/stringtemplate4/org/stringtemplate/v4/StringRenderer.java
- * Dbg  -leave-one-out -java_guava      corpus/java/training/guava     corpus/java/training/guava/base/Absent.java
+ * Dbg  -java_st      corpus/java/training/stringtemplate4/org/stringtemplate/v4/StringRenderer.java
+ * Dbg  -java_guava   corpus/java/training/guava/base/Absent.java
  * Dbg  -java      corpus/java/training/antlr4-tool   corpus/java/training/stringtemplate4/org/stringtemplate/v4/AutoIndentWriter.java
  */
 public class Dbg {
@@ -39,19 +39,14 @@ public static void main(String[] args)
 		throws Exception
 	{
 		if ( args.length<2 ) {
-			System.err.println("Dbg [-leave-one-out] [-java|-java8|-antlr|-sqlite|-tsql] root-dir-of-samples test-file");
+			System.err.println("Dbg [-leave-one-out] [-java|-java8|-antlr|-sqlite|-tsql] test-file");
 		}
 
 		int arg = 0;
-		boolean leaveOneOut = false;
+		boolean leaveOneOut = true;
 		boolean collectAnalysis = true;
-		if ( args[arg].equals("-leave-one-out") ) {
-			leaveOneOut = true;
-			arg++;
-		}
 		String language = args[arg++];
 		language = language.substring(1);
-		String corpusDir = args[arg++];
 		String testFilename = args[arg];
 		String output = "???";
 		InputDocument testDoc = null;
@@ -66,9 +61,9 @@ public static void main(String[] args)
 				break;
 			}
 		}
-		if ( lang!=null && leaveOneOut ) {
+		if ( lang!=null ) {
 			start = System.nanoTime();
-			LeaveOneOutValidator validator = new LeaveOneOutValidator(corpusDir, lang);
+			LeaveOneOutValidator validator = new LeaveOneOutValidator(lang.corpusDir, lang);
 			Triple<Formatter,Float,Float> val = validator.validateOneDocument(testFilename, null, collectAnalysis);
 			testDoc = Tool.parse(testFilename, lang);
 			stop = System.nanoTime();
@@ -96,41 +91,6 @@ public static void main(String[] args)
 			ClassificationAnalysis analysis = new ClassificationAnalysis(testDoc, analysisPerToken);
 			System.out.println(analysis);
 		}
-		else if ( lang!=null ) {
-			Corpus corpus = new Corpus(corpusDir, lang);
-			corpus.train();
-			testDoc = Tool.parse(testFilename, lang);
-			start = System.nanoTime();
-			Formatter formatter = new Formatter(corpus,lang.indentSize);
-			output = formatter.format(testDoc, collectAnalysis);
-			stop = System.nanoTime();
-			analysisPerToken = formatter.getAnalysisPerToken();
-
-			ClassificationAnalysis analysis = new ClassificationAnalysis(testDoc, analysisPerToken);
-			System.out.println(analysis);
-
-			CommonTokenStream original_tokens = Tool.tokenize(testDoc.content, corpus.language.lexerClass);
-			List<Token> wsTokens = filter(original_tokens.getTokens(),
-			                              t -> t.getText().matches("\\s+"));
-			String originalWS = tokenText(wsTokens);
-//			Utils.writeFile("/tmp/spaces1", originalWS);
-//			Utils.writeFile("/tmp/input", testDoc.content);
-//			Utils.writeFile("/tmp/output", output);
-
-			CommonTokenStream formatted_tokens = Tool.tokenize(output, corpus.language.lexerClass);
-			wsTokens = filter(formatted_tokens.getTokens(),
-			                  t -> t.getText().matches("\\s+"));
-			String formattedWS = tokenText(wsTokens);
-//			Utils.writeFile("/tmp/spaces2", formattedWS);
-
-			System.out.println("len orig, formatted="+testDoc.content.length()+", "+output.length());
-			System.out.println("ws len orig, formatted="+originalWS.length()+", "+formattedWS.length());
-
-			float editDistance = normalizedLevenshteinDistance(originalWS, formattedWS);
-			System.out.println("Levenshtein distance of ws: "+editDistance);
-			editDistance = normalizedLevenshteinDistance(testDoc.content, output);
-			System.out.println("Levenshtein distance: "+editDistance);
-		}
 
 		if ( lang!=null ) {
 			controller = new GUIController(analysisPerToken, testDoc, output, lang.lexerClass);

src/org/antlr/codebuff/Trainer.java

@@ -52,10 +52,10 @@ public class Trainer {
 	/** When computing child indexes, we use this value for any child list
 	 *  element other than the first one.  If a parent has just one X child,
 	 *  we use the actual child index. If parent has two or more X children,
-	 *  and we are not the first X, use CHILD_INDEX_LIST_ELEMENT. If first
+	 *  and we are not the first X, use CHILD_INDEX_REPEATED_ELEMENT. If first
 	 *  of two or more X children, use actual child index.
 	 */
-	public static final int CHILD_INDEX_LIST_ELEMENT = 1_111_111_111;
+	public static final int CHILD_INDEX_REPEATED_ELEMENT = 1_111_111_111;
 
 	public static final int LIST_PREFIX         = 0;
 	public static final int LIST_FIRST_ELEMENT  = 1;
@@ -352,7 +352,9 @@ public static int getAlignmentCategory(InputDocument doc, TerminalNode node, int
 			if ( alignInfo.a < indentInfo.a ) {
 				return aligncat(alignInfo.a, alignInfo.b);
 			}
+			// Choose indentation over alignment if both at same ancestor level
 			return indentcat(indentInfo.a, indentInfo.b);
+//			return aligncat(alignInfo.a, alignInfo.b); // Should not use alignment over indentation; manual review of output shows indentation kinda messed up
 		}
 
 		// otherwise just return the align or indent we computed
@@ -968,15 +970,15 @@ public static int getChildIndexOrListMembership(ParseTree t) {
 			List<ParserRuleContext> siblings =
 				((ParserRuleContext)parent).getRuleContexts(((ParserRuleContext)t).getClass());
 			if ( siblings.size()>1 && siblings.indexOf(t)>0 ) {
-				return CHILD_INDEX_LIST_ELEMENT;
+				return CHILD_INDEX_REPEATED_ELEMENT;
 			}
 		}
 		// check to see if we are 2nd or beyond repeated token
 		if ( t instanceof TerminalNode ) {
 			List<TerminalNode> repeatedTokens =
 				((ParserRuleContext) parent).getTokens(((TerminalNode) t).getSymbol().getType());
 			if ( repeatedTokens.size()>1 && repeatedTokens.indexOf(t)>0 ) {
-				return CHILD_INDEX_LIST_ELEMENT;
+				return CHILD_INDEX_REPEATED_ELEMENT;
 			}
 		}