Merge pull request #1854 from idodeclare/feature/improve_here_docs

Perl improvements

Author: tarzanek

src/org/opensolaris/opengrok/analysis/perl/PerlLexHelper.java

@@ -22,6 +22,8 @@
  */
 package org.opensolaris.opengrok.analysis.perl;
 import java.io.IOException;
+import java.util.LinkedList;
+import java.util.Queue;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -31,6 +33,7 @@
 interface PerlLexListener {
     void pushState(int state);
     void popState() throws IOException;
+    void switchState(int state);
     void maybeIntraState();
     void take(String value) throws IOException;
     void takeNonword(String value) throws IOException;
@@ -335,27 +338,89 @@ private void takeWhitespace(String whsp) throws IOException {
     }
 
     /**
-     * Begins a Here-document state, and writes the {@code capture} to output.
+     * Parses a Here-document declaration, and writes the {@code capture} to
+     * output. If the declaration is valid, {@code hereSettings} will have been
+     * appended.
      */
-    public void hop(String capture, boolean nointerp, boolean indented)
-        throws IOException {
+    public void hop(String capture) throws IOException {
+        if (!capture.startsWith("<<")) {
+            throw new IllegalArgumentException("bad HERE: " + capture);
+        }
 
         listener.takeNonword(capture);
+        if (hereSettings == null) hereSettings = new LinkedList<>();
+
+        String remaining = capture;
+        int i = 0;
+        HereDocSettings settings;
+        boolean indented = false;
+        boolean nointerp;
+        String terminator;
+
+        String opener = remaining.substring(0, i + 2);
+        remaining = remaining.substring(opener.length());
+        if (remaining.startsWith("~")) {
+            indented = true;
+            remaining = remaining.substring(1);
+        }
+        remaining = remaining.replaceFirst("^\\s+", "");
+        char c = remaining.charAt(0);
+        switch (c) {
+            case '\'':
+                nointerp = true;
+                remaining = remaining.substring(1);
+                break;
+            case '`':
+            case '\"':
+                nointerp = false;
+                remaining = remaining.substring(1);
+                break;
+            case '\\':
+                c = '\0';
+                nointerp = true;
+                remaining = remaining.substring(1);
+                break;
+            default:
+                c = '\0';
+                nointerp = false;
+                break;
+        }
 
-        hereTerminator = null;
-        Matcher m = HERE_TERMINATOR_MATCH.matcher(capture);
-        if (!m.find()) return;
-        hereTerminator = m.group(0);
+        if (c != '\0') {
+            if ((i = remaining.indexOf(c)) < 1) {
+                terminator = remaining;
+            } else {
+                terminator = remaining.substring(0, i);
+            }
+        } else {
+            Matcher m = HERE_TERMINATOR_MATCH.matcher(remaining);
+            if (!m.find()) return;
+            terminator = m.group(0);
+        }
 
         int state;
         if (nointerp) {
             state = indented ? HEREinxN : HERExN;
         } else {
             state = indented ? HEREin : HERE;
         }
-        listener.maybeIntraState();
-        listener.pushState(state);
-        listener.take(Consts.SS);
+        settings = new HereDocSettings(terminator, state);
+        hereSettings.add(settings);
+    }
+
+    /**
+     * Pushes the first Here-document state if any declarations were parsed, or
+     * else does nothing.
+     * @return true if a Here state was pushed
+     */
+    public boolean maybeStartHere() throws IOException {
+        if (hereSettings != null && hereSettings.size() > 0) {
+            HereDocSettings settings = hereSettings.peek();
+            listener.pushState(settings.state);
+            listener.take(Consts.SS);
+            return true;
+        }
+        return false;
     }
 
     /**
@@ -364,26 +429,29 @@ public void hop(String capture, boolean nointerp, boolean indented)
      * @return true if the quote state ended
      */
     public boolean maybeEndHere(String capture) throws IOException {
-        if (!isHereEnding(capture)) {
-            listener.takeNonword(capture);
+        String trimmed = capture.replaceFirst("^\\s+", "");
+        HereDocSettings settings = hereSettings.peek();
+
+        boolean didZspan = false;
+        if (trimmed.equals(settings.terminator)) {
+            listener.take(Consts.ZS);
+            didZspan = true;
+            hereSettings.remove();
+        }
+
+        listener.takeNonword(capture);
+
+        if (hereSettings.size() > 0) {
+            settings = hereSettings.peek();
+            listener.switchState(settings.state);
+            if (didZspan) listener.take(Consts.SS);
             return false;
         } else {
             listener.popState();
-            listener.take(Consts.ZS);
-            listener.takeNonword(capture);
             return true;
         }
     }
 
-    /**
-     * Gets a value indicating if the Here-document should be ended.
-     * @return true if the quote state should end
-     */
-    public boolean isHereEnding(String capture) {
-        String trimmed = capture.replaceFirst("^\\s+", "");
-        return trimmed.equals(hereTerminator);
-    }
-
     /**
      * Splits a sigil identifier -- where the {@code capture} starts with
      * a sigil and ends in an identifier and where Perl allows whitespace after
@@ -504,7 +572,7 @@ public void specialID(String capture) throws IOException {
     }
 
     private final static Pattern HERE_TERMINATOR_MATCH = Pattern.compile(
-        "[a-zA-Z0-9_]+$");
+        "^[a-zA-Z0-9_]+");
 
     /**
      * When matching a quoting construct like qq[], q(), m//, s```, etc., the
@@ -549,6 +617,19 @@ public void specialID(String capture) throws IOException {
      */
     private boolean waitq;
 
-    /** Stores the terminating identifier for For Here-documents */
-    private String hereTerminator;
+    private Queue<HereDocSettings> hereSettings;
+
+    class HereDocSettings {
+        private final String terminator;
+        private final int state;
+
+        public HereDocSettings(String terminator, int state) {
+            this.terminator = terminator;
+            this.state = state;
+        }
+
+        public String getTerminator() { return terminator; }
+
+        public int getState() { return state; }
+    }
 }

src/org/opensolaris/opengrok/analysis/perl/PerlProductions.lexh

@@ -32,7 +32,7 @@ MaybeWhsp     = {WhspChar}*
 EOL = \r|\n|\r\n
 Identifier = [a-zA-Z_] [a-zA-Z0-9_]*
 Sigils = ("$" | "@" | "%" | "&" | "*")
-WxSigils = [[\W]--[\$\@\%\&\*\"\'\`\#]]
+WxSigils = [[\W]--[\$\@\%\&\*\"\'\`\#\r\n]]
 
 // Perl special identifiers (four of six from
 // https://perldoc.perl.org/perldata.html#Identifier-parsing):
@@ -100,35 +100,38 @@ TRhash = "tr"\#
 TRpunc = "tr" {MaybeWhsp} {Quo0xHash}
 TRword = "tr" {WhiteSpace} \w
 
-HereContinuation = \,{MaybeWhsp} "<<"\~? {MaybeWhsp}
-MaybeHereMarkers = ([\"\'\`\\]?{Identifier} [^\n\r]* {HereContinuation})?
+HereEOF1 = [\"][^\r\n\"]*[\"]
+HereEOF2 = [\`][^\r\n\`]*[\`]
+HereEOF3 = [\'][^\r\n\']*[\']
+HereEOF4 = [\\]?{Identifier}
 
 //
 // Track some keywords that can be used to identify heuristically a possible
 // beginning of the shortcut syntax, //, for m//. Also include any perlfunc
-// that takes /PATTERN/ -- which is just "split". Heuristics using punctuation
-// are defined inline later in some rules.
+// that takes /PATTERN/. Heuristics using punctuation are defined inline later
+// in some rules.
 //
 Mwords_1 = ("eq" | "ne" | "le" | "ge" | "lt" | "gt" | "cmp")
 Mwords_2 = ("if" | "unless" | "or" | "and" | "not")
-Mwords_3 = ("split")
+Mwords_3 = ("split" | "grep")
 Mwords = ({Mwords_1} | {Mwords_2} | {Mwords_3})
 
 Mpunc1YYIN = [\(\!]
-Mpunc2IN = ([!=]"~" | [\:\?\=\+\-\<\>] | "=="|"!="|"<="|">="|"<=>")
+Mpunc2IN = ([!=]"~" | [\:\?\=\+\-\<\>] | "=="|"!="|"<="|">="|"<=>"|"=>")
 
 //
 // There are two dimensions to quoting: "link"-or-not and "interpolate"-or-not.
 // Unfortunately, we cannot control the %state values, so we have to declare
 // a cross-product of states. (Technically, state values are not guaranteed to
 // be unique by jflex, but states that do not have identical rules will have
-// different values. The following four "QUO" states satisfy this difference
-// criterion. Likewise with the four "HERE" states.)
+// different values. The four "QUO" below states satisfy this difference
+// criterion; as likewise do the four "HERE" states.)
 //
 // YYINITIAL : nothing yet parsed or just after a non-quoted [;{}]
 // INTRA : saw content from YYINITIAL but not yet other state or [;{}]
 // SCOMMENT : single-line comment
 // POD : Perl Plain-Old-Documentation
+// FMT : an output record format
 // QUO : quote-like that is OK to match paths|files|URLs|e-mails
 // QUOxN : "" but with no interpolation
 // QUOxL : quote-like that is not OK to match paths|files|URLs|e-mails
@@ -139,20 +142,21 @@ Mpunc2IN = ([!=]"~" | [\:\?\=\+\-\<\>] | "=="|"!="|"<="|">="|"<=>")
 // HERExN : Here-docs with no interpolation
 // HEREin : Indented Here-docs
 // HEREinxN : Indented Here-docs with no interpolation
-// FMT : an output record format
 //
-%state INTRA SCOMMENT POD FMT QUO QUOxN QUOxL QUOxLxN QM HERE HERExN HEREin HEREinxN
+%state INTRA SCOMMENT POD FMT
+%state QUO QUOxN QUOxL QUOxLxN QM
+%state HERE HERExN HEREin HEREinxN
 
 %%
 <HERE, HERExN> {
     ^ {Identifier} / {MaybeWhsp}{EOL}    {
-        if (h.maybeEndHere(yytext())) yyjump(YYINITIAL);
+        h.maybeEndHere(yytext());
     }
 }
 
 <HEREin, HEREinxN> {
     ^ {MaybeWhsp} {Identifier} / {MaybeWhsp}{EOL}    {
-        if (h.maybeEndHere(yytext())) yyjump(YYINITIAL);
+        h.maybeEndHere(yytext());
     }
 }
 
@@ -173,23 +177,8 @@ Mpunc2IN = ([!=]"~" | [\:\?\=\+\-\<\>] | "=="|"!="|"<="|">="|"<=>")
         takeNonword(yytext());
     }
 
- // Following are rules for Here-documents. Stacked multiple here-docs are
- // recognized, but not fully supported, as only the interpolation setting
- // of the first marker will apply to all sections. (The final, second HERE
- // quoting character is not demanded, as it is superfluous for the needs of
- // xref lexing; and leaving it off simplifies parsing.)
-
- "<<"  {MaybeWhsp} {MaybeHereMarkers} [\"\`]?{Identifier}    {
-    h.hop(yytext(), false/*nointerp*/, false/*indented*/);
- }
- "<<~" {MaybeWhsp} {MaybeHereMarkers} [\"\`]?{Identifier}    {
-    h.hop(yytext(), false/*nointerp*/, true/*indented*/);
- }
- "<<"  {MaybeWhsp} {MaybeHereMarkers} [\'\\]{Identifier}    {
-    h.hop(yytext(), true/*nointerp*/, false/*indented*/);
- }
- "<<~" {MaybeWhsp} {MaybeHereMarkers} [\'\\]{Identifier}    {
-    h.hop(yytext(), true/*nointerp*/, true/*indented*/);
+ "<<"[~]? {MaybeWhsp} ({HereEOF1}|{HereEOF2}|{HereEOF3}|{HereEOF4})    {
+    h.hop(yytext());
  }
 
  {Identifier}    {
@@ -293,7 +282,8 @@ Mpunc2IN = ([!=]"~" | [\:\?\=\+\-\<\>] | "=="|"!="|"<="|">="|"<=>")
  }
 
  // FORMAT start
- ^ {MaybeWhsp} "format" ({WhiteSpace} {Identifier})? {MaybeWhsp} "="    {
+ ^ {MaybeWhsp} "format" ({WhiteSpace} {Identifier})? {MaybeWhsp} "=" /
+     {MaybeWhsp}{EOL}    {
     pushState(FMT);
     if (takeAllContent()) {
         // split off the "  format" as `initial' for keyword processing
@@ -399,11 +389,13 @@ Mpunc2IN = ([!=]"~" | [\:\?\=\+\-\<\>] | "=="|"!="|"<="|">="|"<=>")
     }
 }
 
-<QUO, QUOxN, QUOxL, QUOxLxN> {
+<FMT, QUO, QUOxN, QUOxL, QUOxLxN, HERE, HEREin> {
     \\ \S    {
         takeNonword(yytext());
     }
+}
 
+<QUO, QUOxN, QUOxL, QUOxLxN> {
     {Quo0} |
     \w    {
         String capture = yytext();
@@ -449,6 +441,11 @@ Mpunc2IN = ([!=]"~" | [\:\?\=\+\-\<\>] | "=="|"!="|"<="|">="|"<=>")
     takeNonword(yytext());
     take(Consts.ZS);
   }
+
+  {WhiteSpace}{EOL} |
+  {EOL}    {
+    doStartNewLine();
+  }
 }
 
 <FMT> {
@@ -475,24 +472,32 @@ Mpunc2IN = ([!=]"~" | [\:\?\=\+\-\<\>] | "=="|"!="|"<="|">="|"<=>")
 <SCOMMENT> {
   {WhiteSpace}{EOL} |
   {EOL} {
+    String capture = yytext();
+    yypushback(capture.length());
     yypop();
     take(Consts.ZS);
-    doStartNewLine();
   }
 }
 
+<YYINITIAL, INTRA> {
+    {WhiteSpace}{EOL} |
+    {EOL}    {
+        String capture = yytext();
+        if (h.maybeStartHere()) {
+            yypushback(capture.length());
+        } else {
+            doStartNewLine();
+        }
+    }
+}
+
 <YYINITIAL, INTRA, SCOMMENT, POD, FMT, QUO, QUOxN, QUOxL, QUOxLxN,
     HERE, HERExN, HEREin, HEREinxN> {
  [&<>\"\']      {
         maybeIntraState();
         takeNonword(yytext());
  }
 
- {WhiteSpace}{EOL} |
- {EOL}          {
-        doStartNewLine();
- }
-
  // Only one whitespace char at a time or else {WxSigils} can be broken
  {WhspChar}     {
         take(yytext());

src/org/opensolaris/opengrok/analysis/perl/PerlSymbolTokenizer.lex

@@ -55,6 +55,8 @@ super(in);
 
     public void popState() throws IOException { yypop(); }
 
+    public void switchState(int state) { yybegin(state); }
+
     public void take(String value) throws IOException {
         // noop
     }

src/org/opensolaris/opengrok/analysis/perl/PerlXref.lex

@@ -57,6 +57,8 @@ import org.opensolaris.opengrok.web.Util;
 
     public void popState() throws IOException { yypop(); }
 
+    public void switchState(int state) { yybegin(state); }
+
     public void take(String value) throws IOException {
         out.write(value);
     }