Bug fix 3.5/fix parsing unterminated strings (#10156)

* fix parsing of unterminated string literals

* added generated file

* added CHANGELOG entry

* Update CHANGELOG

CHANGELOG

@@ -1,6 +1,16 @@
 v3.5.1 (XXXX-XX-XX)
 -------------------
 
+* Properly report parse errors for extraneous unterminated string literals
+  at the end of AQL query strings. For example, in the query `RETURN 1 "abc`,
+  the `RETURN 1` part was parsed fully, and the `"abc` part at the end was
+  parsed until the EOF and then forgotten. But as the fully parsed tokens
+  `RETURN 1` already form a proper query, the unterminated string literal
+  at the end was not reported as a parse error.
+  This is now fixed for unterminated string literals in double and single
+  quotes as well as unterminated multi-line comments at the end of the query
+  string.
+
 * Fix config directory handling, so we don't trap into UNC path lookups on Windows.
 
 * Ignore symlinks when copying JavaScript files at startup via the option

arangod/Aql/tokens.cpp

@@ -2036,6 +2036,12 @@ YY_RULE_SETUP
   /* newline character inside backtick */
 }
 	YY_BREAK
+case YY_STATE_EOF(BACKTICK):
+{
+  auto parser = yyextra;
+  parser->registerParseError(TRI_ERROR_QUERY_PARSE, "unexpected unterminated identifier", yylloc->first_line, yylloc->first_column);
+}
+	YY_BREAK
 case 68:
 YY_RULE_SETUP
 {
@@ -2074,6 +2080,12 @@ YY_RULE_SETUP
   /* newline character inside forwardtick */
 }
 	YY_BREAK
+case YY_STATE_EOF(FORWARDTICK):
+{
+  auto parser = yyextra;
+  parser->registerParseError(TRI_ERROR_QUERY_PARSE, "unexpected unterminated identifier", yylloc->first_line, yylloc->first_column);
+}
+	YY_BREAK
 case 73:
 YY_RULE_SETUP
 {
@@ -2114,6 +2126,12 @@ YY_RULE_SETUP
   /* newline character inside quote */
 }
 	YY_BREAK
+case YY_STATE_EOF(DOUBLE_QUOTE):
+{
+  auto parser = yyextra;
+  parser->registerParseError(TRI_ERROR_QUERY_PARSE, "unexpected unterminated string literal", yylloc->first_line, yylloc->first_column);
+}
+	YY_BREAK
 case 78:
 YY_RULE_SETUP
 {
@@ -2151,6 +2169,12 @@ YY_RULE_SETUP
   /* newline character inside quote */
 }
 	YY_BREAK
+case YY_STATE_EOF(SINGLE_QUOTE):
+{
+  auto parser = yyextra;
+  parser->registerParseError(TRI_ERROR_QUERY_PARSE, "unexpected unterminated string literal", yylloc->first_line, yylloc->first_column);
+}
+	YY_BREAK
 case 83:
 YY_RULE_SETUP
 {
@@ -2301,6 +2325,12 @@ YY_RULE_SETUP
   // eat the lone star
 }
 	YY_BREAK
+case YY_STATE_EOF(COMMENT_MULTI):
+{
+  auto parser = yyextra;
+  parser->registerParseError(TRI_ERROR_QUERY_PARSE, "unexpected unterminated multi-line comment", yylloc->first_line, yylloc->first_column);
+}
+	YY_BREAK
 case 97:
 /* rule 97 can match eol */
 YY_RULE_SETUP
@@ -2321,12 +2351,7 @@ YY_RULE_SETUP
 ECHO;
 	YY_BREAK
 case YY_STATE_EOF(INITIAL):
-case YY_STATE_EOF(BACKTICK):
-case YY_STATE_EOF(FORWARDTICK):
-case YY_STATE_EOF(SINGLE_QUOTE):
-case YY_STATE_EOF(DOUBLE_QUOTE):
 case YY_STATE_EOF(COMMENT_SINGLE):
-case YY_STATE_EOF(COMMENT_MULTI):
 	yyterminate();
 
 	case YY_END_OF_BUFFER:

arangod/Aql/tokens.ll

@@ -363,6 +363,11 @@ class Parser;
   /* newline character inside backtick */
 }
 
+<BACKTICK><<EOF>> {
+  auto parser = yyextra;
+  parser->registerParseError(TRI_ERROR_QUERY_PARSE, "unexpected unterminated identifier", yylloc->first_line, yylloc->first_column);
+}
+
 <BACKTICK>. {
   /* any character (except newline) inside backtick */
 }
@@ -391,6 +396,11 @@ class Parser;
   /* newline character inside forwardtick */
 }
 
+<FORWARDTICK><<EOF>> {
+  auto parser = yyextra;
+  parser->registerParseError(TRI_ERROR_QUERY_PARSE, "unexpected unterminated identifier", yylloc->first_line, yylloc->first_column);
+}
+
 <FORWARDTICK>. {
   /* any character (except newline) inside forwardtick */
 }
@@ -421,6 +431,11 @@ class Parser;
   /* newline character inside quote */
 }
 
+<DOUBLE_QUOTE><<EOF>> {
+  auto parser = yyextra;
+  parser->registerParseError(TRI_ERROR_QUERY_PARSE, "unexpected unterminated string literal", yylloc->first_line, yylloc->first_column);
+}
+
 <DOUBLE_QUOTE>. {
   /* any character (except newline) inside quote */
 }
@@ -447,6 +462,11 @@ class Parser;
   /* newline character inside quote */
 }
 
+<SINGLE_QUOTE><<EOF>> {
+  auto parser = yyextra;
+  parser->registerParseError(TRI_ERROR_QUERY_PARSE, "unexpected unterminated string literal", yylloc->first_line, yylloc->first_column);
+}
+
 <SINGLE_QUOTE>. {
   /* any character (except newline) inside quote */
 }
@@ -572,6 +592,11 @@ class Parser;
   // eat the lone star
 }
 
+<COMMENT_MULTI><<EOF>> {
+  auto parser = yyextra;
+  parser->registerParseError(TRI_ERROR_QUERY_PARSE, "unexpected unterminated multi-line comment", yylloc->first_line, yylloc->first_column);
+}
+
 <COMMENT_MULTI>\n {
   /* line numbers are counted elsewhere already */
   yycolumn = 0;

tests/js/server/aql/aql-parse.js

@@ -125,6 +125,11 @@ function ahuacatlParseTestSuite () {
       assertParseError(errors.ERROR_QUERY_PARSE.code, "return -");
       assertParseError(errors.ERROR_QUERY_PARSE.code, "return +");
       assertParseError(errors.ERROR_QUERY_PARSE.code, "return ."); 
+      assertParseError(errors.ERROR_QUERY_PARSE.code, "RETURN 1 /* "); 
+      assertParseError(errors.ERROR_QUERY_PARSE.code, "RETURN 1 \" foo "); 
+      assertParseError(errors.ERROR_QUERY_PARSE.code, "RETURN 1 ' foo "); 
+      assertParseError(errors.ERROR_QUERY_PARSE.code, "RETURN 1 `foo "); 
+      assertParseError(errors.ERROR_QUERY_PARSE.code, "RETURN 1 ´foo "); 
     },
 
 ////////////////////////////////////////////////////////////////////////////////