Merge branch 'aql2' of ssh://github.com/triAGENS/ArangoDB into aql2

.gitignore

@@ -70,8 +70,15 @@ UnitTests/geo_suite
 arangod/Ahuacatl/ahuacatl-grammar.c
 arangod/Ahuacatl/ahuacatl-grammar.cpp
 arangod/Ahuacatl/ahuacatl-grammar.h
+arangod/Ahuacatl/ahuacatl-grammar.hpp
 arangod/Ahuacatl/ahuacatl-grammar.output
 arangod/Ahuacatl/ahuacatl-tokens.cpp
+arangod/Aql/grammar.c
+arangod/Aql/grammar.cpp
+arangod/Aql/grammar.h
+arangod/Aql/grammar.hpp
+arangod/Aql/grammar.output
+arangod/Aql/tokens.cpp
 bin/
 etc/arangodb/*.conf
 js/common/bootstrap/errors.js

arangod/Aql/Parser.cpp

@@ -39,7 +39,16 @@ using namespace triagens::aql;
 /// @brief create the parser
 ////////////////////////////////////////////////////////////////////////////////
 
-Parser::Parser () {
+Parser::Parser (Query* query)
+  : _query(query),
+    _scanner(nullptr),
+    _buffer(query->queryString()),
+    _remainingLength(query->queryLength()),
+    _offset(0),
+    _marker(nullptr) {
+  
+  //Aqlllex_init(&context->_parser->_scanner);
+  //Aqlset_extra(context, context->_parser->_scanner);
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -47,6 +56,7 @@ Parser::Parser () {
 ////////////////////////////////////////////////////////////////////////////////
 
 Parser::~Parser () {
+//  Aqllex_destroy(parser->_scanner);
 }
 
 // -----------------------------------------------------------------------------

arangod/Aql/Parser.h

@@ -31,10 +31,13 @@
 #define ARANGODB_AQL_PARSER_H 1
 
 #include "Basics/Common.h"
+#include "Aql/Query.h"
 
 namespace triagens {
   namespace aql {
 
+    class Query;
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                      class Parser
 // -----------------------------------------------------------------------------
@@ -55,7 +58,7 @@ namespace triagens {
 /// @brief create the parser
 ////////////////////////////////////////////////////////////////////////////////
 
-        Parser ();
+        Parser (Query*);
 
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief destroy the parser
@@ -69,23 +72,105 @@ namespace triagens {
 
       public:
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief return the scanner
+////////////////////////////////////////////////////////////////////////////////
+        
+        inline void* scanner () const {
+          return _scanner;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief a pointer to the start of the query string
+////////////////////////////////////////////////////////////////////////////////
+        
+        inline char const* queryString () const {
+          return _query->queryString();
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief return the remaining length of the query string to process
+////////////////////////////////////////////////////////////////////////////////
+
+        inline size_t remainingLength () const {
+          return _remainingLength;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief return the current marker position
+////////////////////////////////////////////////////////////////////////////////
+
+        inline char const* marker () const {
+          return _marker;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief set the current marker position
+////////////////////////////////////////////////////////////////////////////////
+
+        inline void marker (char const* marker) {
+          _marker = marker;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief return the current parse position
+////////////////////////////////////////////////////////////////////////////////
+        
+        inline size_t offset () const {
+          return _offset;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief adjust the current parse position
+////////////////////////////////////////////////////////////////////////////////
+
+        inline void increaseOffset (int offset) {
+          _offset += (size_t) offset;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief fill the output buffer with a fragment of the query
+////////////////////////////////////////////////////////////////////////////////
+
+        void fillBuffer (char* result,
+                         size_t length) {
+          memcpy(result, _buffer, length);
+          _buffer += length;
+          _remainingLength -= length;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief register a string
+////////////////////////////////////////////////////////////////////////////////
+
+        char* registerString (char const* p, 
+                              size_t length,
+                              bool mustEscape) {
+          // TODO
+          return nullptr;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief set the query type
+////////////////////////////////////////////////////////////////////////////////
+
+        void type (QueryType type) {
+          _query->type(type);
+        }
+
 // -----------------------------------------------------------------------------
 // --SECTION--                                                 private variables
 // -----------------------------------------------------------------------------
 
       private:
 
-        void* _scanner; // the lexer generated by flex
-        char* _buffer;  // the currently procssed part of the query string
-        size_t _length; // remaining length of the query string, modified during parsing
-        size_t _queryLength; // overall length of the query string, not modified
+        Query*        _query;           // the query object
 
-
-        // TODO: check if these are still required
-        // TRI_vector_pointer_t _scopes;
-        // TRI_vector_pointer_t _stack;
-        // char* _marker; // used temporarily during parsing
-        // size_t _offset; // current parse position
+        void*         _scanner; // the lexer generated by flex
+        char const*   _buffer;          // the currently processed part of the query string
+        size_t        _remainingLength; // remaining length of the query string, modified during parsing
+        size_t        _offset;          // current parse position
+        char const*   _marker;          // a position used temporarily during parsing
     };
 
   }

arangod/Aql/Query.cpp

@@ -49,7 +49,7 @@ Query::Query (TRI_vocbase_t* vocbase,
   : _vocbase(vocbase),
     _queryString(queryString),
     _queryLength(queryLength),
-    _queryType(QUERY_READ),
+    _type(AQL_QUERY_READ),
     _bindParameters(bindParameters),
     _error() {
 }
@@ -101,7 +101,7 @@ void Query::explain () {
 ////////////////////////////////////////////////////////////////////////////////
 
 void Query::parseQuery () {
-  Parser parser;
+  Parser parser(this);
 }
 
 /*

arangod/Aql/Query.h

@@ -48,11 +48,11 @@ namespace triagens {
 ////////////////////////////////////////////////////////////////////////////////
 
     enum QueryType {
-      QUERY_READ,
-      QUERY_REMOVE,
-      QUERY_INSERT,
-      QUERY_UPDATE,
-      QUERY_REPLACE
+      AQL_QUERY_READ,
+      AQL_QUERY_REMOVE,
+      AQL_QUERY_INSERT,
+      AQL_QUERY_UPDATE,
+      AQL_QUERY_REPLACE
     };
 
 // -----------------------------------------------------------------------------
@@ -84,6 +84,38 @@ namespace triagens {
 
       public:
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief get the query type
+////////////////////////////////////////////////////////////////////////////////
+
+        inline QueryType type () const {
+          return _type;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief set the query type
+////////////////////////////////////////////////////////////////////////////////
+
+        void type (QueryType type) {
+          _type = type;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief get the query string
+////////////////////////////////////////////////////////////////////////////////
+
+        char const* queryString () const {
+          return _queryString;
+        }
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief get the length of the query string
+////////////////////////////////////////////////////////////////////////////////
+
+        size_t queryLength () const {
+          return _queryLength;
+        }
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief execute an AQL query - TODO: implement and determine return type
 ////////////////////////////////////////////////////////////////////////////////
@@ -124,7 +156,7 @@ namespace triagens {
 
         char const*            _queryString;
         size_t const           _queryLength;
-        QueryType              _queryType;
+        QueryType              _type;
 
         struct TRI_json_s*     _bindParameters;
 

arangod/Aql/tokens.ll

@@ -0,0 +1,427 @@
+
+%option reentrant
+%option 8bit
+%option prefix="Aql"
+%option bison-locations
+%option bison-bridge
+%option yylineno
+%option noyywrap nounput batch
+%x BACKTICK
+%x SINGLE_QUOTE
+%x DOUBLE_QUOTE
+%x COMMENT_SINGLE
+%x COMMENT_MULTI
+
+%{
+#include "Basics/Common.h"
+
+// introduce the namespace here, otherwise following references to 
+// the namespace in auto-generated headers might fail
+
+namespace triagens {
+  namespace aql {
+    class Query;
+    class Parser;
+  }
+}
+
+
+#include "Aql/grammar.h"
+#include "Aql/Parser.h"
+
+#define YY_EXTRA_TYPE triagens::aql::Parser*
+
+#define YY_USER_ACTION yylloc->first_line = yylineno; yylloc->first_column = yycolumn; yylloc->last_column = (int) (yycolumn + yyleng - 1); yycolumn += (int) yyleng; yyextra->increaseOffset(yyleng);
+
+#define YY_NO_INPUT 1
+
+#define YY_INPUT(resultBuffer, resultState, maxBytesToRead) {            \
+  size_t length = yyextra->remainingLength();                            \
+  if (length > maxBytesToRead) {                                         \
+    length = maxBytesToRead;                                             \
+  }                                                                      \
+  if (length > 0) {                                                      \
+    yyextra->fillBuffer(resultBuffer, length);                           \
+    resultState = length;                                                \
+  }                                                                      \
+  else {                                                                 \
+    resultState = YY_NULL;                                               \
+  }                                                                      \
+}
+%}
+
+%%
+
+ /* ---------------------------------------------------------------------------
+  * language keywords
+  * --------------------------------------------------------------------------- */
+
+(?i:FOR) {
+  return T_FOR;
+}
+
+(?i:LET) {
+  return T_LET;
+}
+
+(?i:FILTER) {
+  return T_FILTER;
+}
+
+(?i:RETURN) {
+  return T_RETURN;
+}
+
+(?i:COLLECT) {
+  return T_COLLECT;
+}
+
+(?i:SORT) {
+  return T_SORT;
+}
+
+(?i:LIMIT) { 
+  return T_LIMIT;
+}
+
+(?i:ASC) {
+  return T_ASC;
+}
+
+(?i:DESC) {
+  return T_DESC;
+}
+
+(?i:IN) {
+  return T_IN;
+}
+
+(?i:INTO) {
+  return T_INTO;
+}
+
+(?i:WITH) {
+  return T_WITH;
+}
+
+(?i:REMOVE) {
+  return T_REMOVE;
+}
+
+(?i:INSERT) {
+  return T_INSERT;
+}
+
+(?i:UPDATE) {
+  return T_UPDATE;
+}
+
+(?i:REPLACE) {
+  return T_REPLACE;
+}
+
+ /* ---------------------------------------------------------------------------
+  * predefined type literals
+  * --------------------------------------------------------------------------- */
+
+(?i:NULL) {
+  return T_NULL;
+}
+
+(?i:TRUE) {
+  return T_TRUE;
+}
+
+(?i:FALSE) {
+  return T_FALSE;
+}
+ 
+ /* ---------------------------------------------------------------------------
+  * operators
+  * --------------------------------------------------------------------------- */
+
+"==" {
+  return T_EQ;
+}
+
+"!=" {
+  return T_NE;
+}
+
+">=" {
+  return T_GE;
+}
+
+">" {
+  return T_GT;
+}
+
+"<=" {
+  return T_LE;
+}
+
+"<" {
+  return T_LT;
+}
+
+"=" {
+  return T_ASSIGN;
+}
+
+"!" {
+  return T_NOT;
+}
+
+"&&" {
+  return T_AND;
+}
+
+"||" {
+  return T_OR;
+}
+
+"+" {
+  return T_PLUS;
+}
+
+"-" {
+  return T_MINUS;
+}
+
+"*" {
+  return T_TIMES;
+}
+
+"/" {
+  return T_DIV;
+}
+
+"%" {
+  return T_MOD;
+}
+
+"?" {
+  return T_QUESTION;
+}
+
+"::" {
+  return T_SCOPE;
+}
+
+":" {
+  return T_COLON;
+}
+
+"[*]" {
+  return T_EXPAND;
+}
+
+".." {
+  return T_RANGE; 
+}
+ 
+ /* ---------------------------------------------------------------------------
+  * punctuation
+  * --------------------------------------------------------------------------- */
+
+"," {
+  return T_COMMA;
+}
+
+"(" {
+  return T_OPEN;
+}
+
+")" {
+  return T_CLOSE;
+}
+
+"{" {
+  return T_DOC_OPEN;
+}
+
+"}" {
+  return T_DOC_CLOSE;
+}
+
+"[" {
+  return T_LIST_OPEN;
+}
+
+"]" {
+  return T_LIST_CLOSE;
+}
+ 
+ /* ---------------------------------------------------------------------------
+  * literals
+  * --------------------------------------------------------------------------- */
+
+([a-zA-Z][_a-zA-Z0-9]*|_+[a-zA-Z]+[_a-zA-Z0-9]*) { 
+  /* unquoted string */
+  yylval->strval = yyextra->registerString(yytext, yyleng, false);
+  return T_STRING; 
+}
+
+<INITIAL>` {
+  /* string enclosed in backticks */
+  yyextra->marker(yyextra->queryString() + yyextra->offset());
+  BEGIN(BACKTICK);
+}
+
+<BACKTICK>` {
+  /* end of backtick-enclosed string */
+  BEGIN(INITIAL);
+  yylval->strval = yyextra->registerString(yyextra->marker(), yyextra->offset() - (yyextra->marker() - yyextra->queryString()) - 1, true);
+  return T_STRING;
+}
+
+<BACKTICK>\\. {
+  /* character escaped by backslash */
+  BEGIN(BACKTICK);
+}
+
+<BACKTICK>\n {
+  /* newline character inside backtick */
+  BEGIN(BACKTICK);
+}
+
+<BACKTICK>. {
+  /* any character (except newline) inside backtick */
+  BEGIN(BACKTICK);
+}
+
+<INITIAL>\" {
+  yyextra->marker(yyextra->queryString() + yyextra->offset());
+  BEGIN(DOUBLE_QUOTE);
+}
+
+<DOUBLE_QUOTE>\" {
+  /* end of quote-enclosed string */
+  BEGIN(INITIAL);
+  yylval->strval = yyextra->registerString(yyextra->marker(), yyextra->offset() - (yyextra->marker() - yyextra->queryString()) - 1, true);
+  return T_QUOTED_STRING;
+}
+
+<DOUBLE_QUOTE>\\. {
+  /* character escaped by backslash */
+  BEGIN(DOUBLE_QUOTE);
+}
+
+<DOUBLE_QUOTE>\n {
+  /* newline character inside quote */
+  BEGIN(DOUBLE_QUOTE);
+}
+
+<DOUBLE_QUOTE>. {
+  /* any character (except newline) inside quote */
+  BEGIN(DOUBLE_QUOTE);
+}
+
+<INITIAL>' {
+  yyextra->marker(yyextra->queryString() + yyextra->offset());
+  BEGIN(SINGLE_QUOTE);
+}
+
+<SINGLE_QUOTE>' {
+  /* end of quote-enclosed string */
+  BEGIN(INITIAL);
+  yylval->strval = yyextra->registerString(yyextra->marker(), yyextra->offset() - (yyextra->marker() - yyextra->queryString()) - 1, true);
+  return T_QUOTED_STRING;
+}
+
+<SINGLE_QUOTE>\\. {
+  /* character escaped by backslash */
+  BEGIN(SINGLE_QUOTE);
+}
+
+<SINGLE_QUOTE>\n {
+  /* newline character inside quote */
+  BEGIN(SINGLE_QUOTE);
+}
+
+<SINGLE_QUOTE>. {
+  /* any character (except newline) inside quote */
+  BEGIN(SINGLE_QUOTE);
+}
+
+(0|[1-9][0-9]*) {  
+  /* a numeric integer value */
+  yylval->strval = yyextra->registerString(yytext, yyleng, false); 
+  return T_INTEGER;
+}
+
+(0|[1-9][0-9]*)(\.[0-9]+([eE]([\-\+])?[0-9]+)?) {  
+  /* a numeric double value */
+  yylval->strval = yyextra->registerString(yytext, yyleng, false); 
+  return T_DOUBLE;
+}
+
+ /* ---------------------------------------------------------------------------
+  * bind parameters
+  * --------------------------------------------------------------------------- */
+
+@@?[a-zA-Z0-9][a-zA-Z0-9_]* {
+  /* bind parameters must start with a @
+     if followed by another @, this is a collection name parameter */
+  yylval->strval = yyextra->registerString(yytext + 1, yyleng - 1, false); 
+  return T_PARAMETER;
+}
+
+ /* ---------------------------------------------------------------------------
+  * whitespace etc.
+  * --------------------------------------------------------------------------- */
+
+[ \t\r\n]+ {
+  /* whitespace is ignored */ 
+}
+
+<INITIAL>"//" {
+  BEGIN(COMMENT_SINGLE);
+}
+
+<COMMENT_SINGLE>\r?\n {
+  yylineno++;
+  BEGIN(INITIAL);
+}
+
+<COMMENT_SINGLE>\r {
+  yylineno++;
+  BEGIN(INITIAL);
+}
+
+<COMMENT_SINGLE>[^*\r\n]+ { 
+  // eat comment in chunks
+}
+
+<INITIAL>"/*" {
+  BEGIN(COMMENT_MULTI);
+}
+
+<COMMENT_MULTI>"*/" {
+  BEGIN(INITIAL);
+}
+
+<COMMENT_MULTI>[^*\r\n]+ { 
+  // eat comment in chunks
+}
+
+<COMMENT_MULTI>"*" {
+  // eat the lone star
+}
+
+<COMMENT_MULTI>\r?\n {
+  yylineno++;
+}
+
+<COMMENT_MULTI>\r {
+  yylineno++;
+}
+
+ 
+. {
+  /* anything else is returned as it is */
+  return (int) yytext[0];
+}
+
+%%
+

arangod/CMakeLists.txt

@@ -58,9 +58,11 @@ add_executable(
     Ahuacatl/ahuacatl-tokens.cpp
     Ahuacatl/ahuacatl-variable.cpp
     Aql/ExecutionPlan.cpp
+    Aql/grammar.cpp
     Aql/Parser.cpp
     Aql/Query.cpp
     Aql/QueryError.cpp
+    Aql/tokens.cpp
     BitIndexes/bitarray.cpp
     BitIndexes/bitarrayIndex.cpp
     CapConstraint/cap-constraint.cpp

arangod/Makefile.files

@@ -39,13 +39,15 @@ arangod_libarangod_a_SOURCES = \
 	arangod/Ahuacatl/ahuacatl-tokens.cpp \
 	arangod/Ahuacatl/ahuacatl-variable.cpp \
 	arangod/Aql/ExecutionPlan.cpp \
+	arangod/Aql/grammar.cpp \
 	arangod/Aql/Parser.cpp \
 	arangod/Aql/Query.cpp \
 	arangod/Aql/QueryError.cpp \
+	arangod/Aql/tokens.cpp \
 	arangod/BitIndexes/bitarray.cpp \
 	arangod/BitIndexes/bitarrayIndex.cpp \
 	arangod/CapConstraint/cap-constraint.cpp \
-        arangod/Cluster/AgencyComm.cpp \
+	arangod/Cluster/AgencyComm.cpp \
 	arangod/Cluster/ApplicationCluster.cpp \
 	arangod/Cluster/ClusterComm.cpp \
 	arangod/Cluster/ClusterInfo.cpp \
@@ -166,20 +168,25 @@ endif
 ################################################################################
 
 FLEXXX_FILES += \
-        arangod/Ahuacatl/ahuacatl-tokens.cpp
+	arangod/Ahuacatl/ahuacatl-tokens.cpp \
+	arangod/Aql/tokens.cpp
 
 ################################################################################
 ### @brief bison
 ################################################################################
 
 BISON_FILES += \
-	arangod/Ahuacatl/ahuacatl-grammar.cpp
+	arangod/Ahuacatl/ahuacatl-grammar.cpp \
+	arangod/Aql/grammar.cpp
 
 if ENABLE_MAINTAINER_MODE
 
 CLEANUP += \
 	arangod/Ahuacatl/ahuacatl-grammar.h \
-	arangod/Ahuacatl/ahuacatl-grammar.cpp
+	arangod/Ahuacatl/ahuacatl-grammar.cpp \
+	arangod/Aql/grammar.h \
+	arangod/Aql/grammar.cpp
+
 endif
 
 ################################################################################