Create only one antlr token per text instead of separate tokens for newlines and spaces.

Adjusted whitespace control accordingly.

Author: fsmike64

handlebars/src/main/antlr4/com/github/jknack/handlebars/internal/HbsLexer.g4

@@ -15,22 +15,37 @@ lexer grammar HbsLexer;
     this.end = end;
   }
 
-  private boolean isWhite(int ch) {
-    return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n';
-  }
-
   private boolean consumeUntil(final String token) {
-    int offset = 0;
-    while(!isEOF(offset) && !(ahead("\\" + token, offset) || ahead(token, offset)) &&
-      !isWhite(_input.LA(offset + 1))) {
-      offset+=1;
+    int tokenOffset = 0;
+    int colOffset = 0;
+    int lineOffset = 0;
+    boolean inNewline = false;
+    boolean resetLine = false;
+    while(!isEOF(tokenOffset) && !(ahead("\\" + token, tokenOffset) || ahead(token, tokenOffset))) {
+      if (resetLine) {
+        colOffset = 0;
+        lineOffset+=1;
+        resetLine = false;
+      }
+      tokenOffset+=1;
+      colOffset+=1;
+      int chr = _input.LA(tokenOffset);
+      if (!inNewline && '\n' == chr) {
+        resetLine = true;
+      }
+      inNewline = false;
+      if ('\r' == chr) {
+        inNewline = true;
+        resetLine = true;
+      }
     }
-    if (offset == 0) {
+    if (tokenOffset == 0) {
       return false;
     }
     // Since we found the text, increase the CharStream's index.
-    _input.seek(_input.index() + offset - 1);
-    getInterpreter().setCharPositionInLine(_tokenStartCharPositionInLine + offset - 1);
+    _input.seek(_input.index() + tokenOffset - 1);
+    getInterpreter().setCharPositionInLine(_tokenStartCharPositionInLine + colOffset - 1);
+    getInterpreter().setLine(_tokenStartLine + lineOffset);
     return true;
   }
 
@@ -205,17 +220,6 @@ START
  : {startToken(start)}? . -> pushMode(VAR)
  ;
 
-SPACE
- :
-  [ \t]+
- ;
-
-NL
- :
-   '\r'? '\n'
- | '\r'
- ;
-
 mode SET_DELIMS;
 
 END_DELIM

handlebars/src/main/antlr4/com/github/jknack/handlebars/internal/HbsParser.g4

@@ -46,9 +46,7 @@ body
 
 statement
   :
-    spaces
-  | newline
-  | text
+    text
   | block
   | var
   | tvar
@@ -72,14 +70,6 @@ text
     TEXT
   ;
 
-spaces
-  : SPACE
-  ;
-
-newline
-  : NL
-  ;
-
 block
   :
     startToken = START_BLOCK DECORATOR? sexpr blockParams? END

handlebars/src/main/java/com/github/jknack/handlebars/internal/BaseTemplate.java

@@ -76,11 +76,6 @@ abstract class BaseTemplate implements Template {
    */
   protected String filename;
 
-  /**
-   * A Handlebars.js lock.
-   */
-  private final Object jsLock = new Object();
-
   /**
    * A pre-compiled JavaScript function.
    */

handlebars/src/main/java/com/github/jknack/handlebars/internal/HbsParserFactory.java

@@ -90,10 +90,10 @@ public Template parse(final TemplateSource source) throws IOException {
           logger.debug("Applying Mustache spec");
           new ParseTreeWalker().walk(new MustacheSpec(), tree);
         }
-
+        
         if (lexer.whiteSpaceControl) {
           logger.debug("Applying white spaces control");
-          new ParseTreeWalker().walk(new WhiteSpaceControl(), tree);
+          new ParseTreeWalker().walk(new WhiteSpaceControl((CommonTokenStream)parser.getTokenStream()), tree);
         }
 
         /**

handlebars/src/main/java/com/github/jknack/handlebars/internal/MustacheSpec.java

@@ -22,14 +22,13 @@
 
 import org.antlr.v4.runtime.CommonToken;
 import org.antlr.v4.runtime.Token;
+import org.apache.commons.lang3.StringUtils;
 
 import com.github.jknack.handlebars.internal.HbsParser.AmpvarContext;
 import com.github.jknack.handlebars.internal.HbsParser.BlockContext;
 import com.github.jknack.handlebars.internal.HbsParser.CommentContext;
 import com.github.jknack.handlebars.internal.HbsParser.DelimitersContext;
-import com.github.jknack.handlebars.internal.HbsParser.NewlineContext;
 import com.github.jknack.handlebars.internal.HbsParser.PartialContext;
-import com.github.jknack.handlebars.internal.HbsParser.SpacesContext;
 import com.github.jknack.handlebars.internal.HbsParser.TemplateContext;
 import com.github.jknack.handlebars.internal.HbsParser.TextContext;
 import com.github.jknack.handlebars.internal.HbsParser.TvarContext;
@@ -45,65 +44,51 @@
 public class MustacheSpec extends HbsParserBaseListener {
 
   /**
-   * Track if the current line has real text (not spaces).
+   * Tracks if the current line should be treated as stand-alone.
    */
-  private boolean nonSpace = false;
+  private Boolean standAlone;
 
   /**
-   * Track if the current line has mustache instruction.
+   * Tracks text tokens for future whitespace removal.
    */
-  private Boolean hasTag;
+  private List<CommonToken> textTokens = new ArrayList<>();
 
   /**
-   * Track the current line.
+   * Channel for tokens that need their last line removed.
    */
-  protected StringBuilder line = new StringBuilder();
-
-  /**
-   * Track the spaces/lines that need to be excluded.
-   */
-  private List<CommonToken> spaces = new ArrayList<>();
-
-  @Override
-  public void enterSpaces(final SpacesContext ctx) {
-    CommonToken space = (CommonToken) ctx.SPACE().getSymbol();
-    line.append(space.getText());
-    spaces.add(space);
-  }
-
-  @Override
-  public void enterNewline(final NewlineContext ctx) {
-    CommonToken newline = (CommonToken) ctx.NL().getSymbol();
-    spaces.add(newline);
-    stripSpaces();
-  }
+  public static final int REMOVE_LAST_LINE_CHANNEL = Token.MIN_USER_CHANNEL_VALUE;
 
   @Override
   public void exitTemplate(final TemplateContext ctx) {
-    stripSpaces();
+    removeWhitespace();
+    this.textTokens.clear();
+    this.standAlone = null;
   }
 
-  /**
-   * Move tokens to the hidden channel if necessary.
-   */
-  private void stripSpaces() {
-    boolean hasTag = this.hasTag == null ? false : this.hasTag.booleanValue();
-    if (hasTag && !nonSpace) {
-      for (CommonToken space : spaces) {
-        space.setChannel(Token.HIDDEN_CHANNEL);
+  @Override
+  public void enterText(final TextContext ctx) {
+    CommonToken currentToken = (CommonToken) ctx.getStart();
+    String currentText = currentToken.getText();
+    Integer secondLineIndex = MustacheStringUtils.indexOfSecondLine(currentText);
+    if (secondLineIndex == null) {
+      // Non-whitespace was found. Clear list and start again with the current token
+      this.textTokens.clear();
+      this.standAlone = null;
+    } else if (secondLineIndex >= 0) {
+      // Try to remove whitespace from the previously saved text tokens
+      boolean canRemoveWhitespace = removeWhitespace();
+      if (canRemoveWhitespace) {
+        // Remove the first line of this text as well
+        String newText = StringUtils.substring(currentText, secondLineIndex);
+        currentToken.setText(newText);
       }
-    } else {
-      spaces.clear();
+
+      // Clear list and start again with the current token
+      this.textTokens.clear();
+      this.standAlone = null;
     }
 
-    this.hasTag = null;
-    nonSpace = false;
-    line.setLength(0);
-  }
-
-  @Override
-  public void enterText(final TextContext ctx) {
-    nonSpace = true;
+    this.textTokens.add(currentToken);
   }
 
   @Override
@@ -136,7 +121,8 @@ public void enterUnless(final UnlessContext ctx) {
     hasTag(true);
   }
 
-  @Override public void enterElseBlock(final HbsParser.ElseBlockContext ctx) {
+  @Override
+  public void enterElseBlock(final HbsParser.ElseBlockContext ctx) {
     hasTag(true);
   }
 
@@ -166,8 +152,63 @@ public void enterVar(final VarContext ctx) {
    * @param hasTag True, to indicate there is a mustache instruction.
    */
   private void hasTag(final boolean hasTag) {
-    if (this.hasTag != Boolean.FALSE) {
-      this.hasTag = hasTag;
+    if (this.standAlone != Boolean.FALSE) {
+      this.standAlone = hasTag;
     }
   }
+
+  /**
+   * Remove whitespace from previously saved text tokens.
+   *
+   * @return True if whitespace could be removed. False otherwise.
+   */
+  private boolean removeWhitespace() {
+    boolean canRemoveWhitespace = this.standAlone == null ? false : this.standAlone.booleanValue();
+    if (!canRemoveWhitespace) {
+      return false;
+    }
+
+    if (textTokens.isEmpty()) {
+      return true;
+    }
+
+    // Try to remove whitespace from the last line
+    CommonToken lastToken = textTokens.get(textTokens.size() - 1);
+    String lastText = lastToken.getText();
+    int newlineIndex = StringUtils.lastIndexOfAny(lastText, "\r", "\n");
+    if (newlineIndex >= 0) {
+      String lastLine = lastText.substring(newlineIndex + 1);
+      if (!StringUtils.isWhitespace(lastLine)) {
+        // Cannot remove anything since line contains non-whitespace
+        return false;
+      }
+
+      // Mark the last line for removal
+      lastToken.setChannel(REMOVE_LAST_LINE_CHANNEL);
+    } else {
+      // Check for non-whitespace
+      int maxIndex = textTokens.size() - 1;
+      for (int i = maxIndex; i >= 0; i--) {
+        CommonToken loopToken = textTokens.get(i);
+        String loopText = loopToken.getText();
+        if (!StringUtils.isWhitespace(loopText)) {
+          // Cannot remove anything since line contains non-whitespace
+          return false;
+        }
+        if (i == 0) {
+          // Mark tokens already checked for removal
+          int j = 0;
+          while (j <= maxIndex) {
+            CommonToken whiteSpaceToken = textTokens.get(j);
+            whiteSpaceToken.setChannel(Token.HIDDEN_CHANNEL);
+            j++;
+          }
+
+          break;
+        }
+      }
+    }
+
+    return true;
+  }
 }