////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Dr. Frank Celler
////////////////////////////////////////////////////////////////////////////////

#include <set>

#include "MMFilesSkiplistIndex.h"
#include "Aql/AstNode.h"
#include "Aql/SortCondition.h"
#include "Basics/AttributeNameParser.h"
#include "Basics/Exceptions.h"
#include "Basics/FixedSizeAllocator.h"
#include "Basics/StaticStrings.h"
#include "Basics/VelocyPackHelper.h"
#include "Indexes/SortedIndexAttributeMatcher.h"
#include "MMFiles/MMFilesCollection.h"
#include "MMFiles/MMFilesIndexLookupContext.h"
#include "StorageEngine/PhysicalCollection.h"
#include "Transaction/Helpers.h"
#include "Transaction/Methods.h"
#include "VocBase/LogicalCollection.h"

#include <velocypack/Iterator.h>
#include <velocypack/velocypack-aliases.h>

using namespace arangodb;

// .............................................................................
// recall for all of the following comparison functions:
//
// left < right  return -1
// left > right  return  1
// left == right return  0
//
// furthermore:
//
// the following order is currently defined for placing an order on documents
// undef < null < boolean < number < strings < lists < hash arrays
// note: undefined will be treated as NULL pointer not NULL JSON OBJECT
// within each type class we have the following order
// boolean: false < true
// number: natural order
// strings: lexicographical
// lists: lexicographically and within each slot according to these rules.
// ...........................................................................

/// @brief compares a key with an element, version with proper types
static int CompareKeyElement(void* userData, VPackSlice const* left,
                             MMFilesSkiplistIndexElement const* right, size_t rightPosition) {
  MMFilesIndexLookupContext* context = static_cast<MMFilesIndexLookupContext*>(userData);
  TRI_ASSERT(nullptr != left);
  TRI_ASSERT(nullptr != right);
  return arangodb::basics::VelocyPackHelper::compare(*left, right->slice(context, rightPosition),
                                                     true);
}

/// @brief compares elements, version with proper types
static int CompareElementElement(void* userData, MMFilesSkiplistIndexElement const* left,
                                 size_t leftPosition,
                                 MMFilesSkiplistIndexElement const* right,
                                 size_t rightPosition) {
  MMFilesIndexLookupContext* context = static_cast<MMFilesIndexLookupContext*>(userData);
  TRI_ASSERT(nullptr != left);
  TRI_ASSERT(nullptr != right);

  VPackSlice l = left->slice(context, leftPosition);
  VPackSlice r = right->slice(context, rightPosition);
  return arangodb::basics::VelocyPackHelper::compare(l, r, true);
}

bool MMFilesBaseSkiplistLookupBuilder::isEquality() const {
  return _isEquality;
}

VPackSlice const* MMFilesBaseSkiplistLookupBuilder::getLowerLookup() const {
  return &_lowerSlice;
}

bool MMFilesBaseSkiplistLookupBuilder::includeLower() const {
  return _includeLower;
}

VPackSlice const* MMFilesBaseSkiplistLookupBuilder::getUpperLookup() const {
  return &_upperSlice;
}

bool MMFilesBaseSkiplistLookupBuilder::includeUpper() const {
  return _includeUpper;
}

MMFilesSkiplistLookupBuilder::MMFilesSkiplistLookupBuilder(
    transaction::Methods* trx,
    std::vector<std::vector<arangodb::aql::AstNode const*>>& ops,
    arangodb::aql::Variable const* var, bool reverse)
    : MMFilesBaseSkiplistLookupBuilder(trx) {
  _lowerBuilder->openArray();
  if (ops.empty()) {
    // We only use this skiplist to sort. use empty array for lookup
    _lowerBuilder->close();
    _lowerSlice = _lowerBuilder->slice();
    _upperSlice = _lowerBuilder->slice();
    return;
  }

  auto const& last = ops.back();
  TRI_ASSERT(!last.empty());

  std::pair<arangodb::aql::Variable const*, std::vector<arangodb::basics::AttributeName>> paramPair;

  if (last[0]->type != arangodb::aql::NODE_TYPE_OPERATOR_BINARY_EQ &&
      last[0]->type != arangodb::aql::NODE_TYPE_OPERATOR_BINARY_IN) {
    _isEquality = false;
    _upperBuilder->openArray();
    for (size_t i = 0; i < ops.size() - 1; ++i) {
      auto const& oplist = ops[i];
      TRI_ASSERT(oplist.size() == 1);
      auto const& op = oplist[0];
      TRI_ASSERT(op->type == arangodb::aql::NODE_TYPE_OPERATOR_BINARY_EQ ||
                 op->type == arangodb::aql::NODE_TYPE_OPERATOR_BINARY_IN);
      TRI_ASSERT(op->numMembers() == 2);
      auto value = op->getMember(0);
      if (value->isAttributeAccessForVariable(paramPair) && paramPair.first == var) {
        value = op->getMember(1);
        TRI_ASSERT(!(value->isAttributeAccessForVariable(paramPair) && paramPair.first == var));
      }
      value->toVelocyPackValue(*(_lowerBuilder.get()));
      value->toVelocyPackValue(*(_upperBuilder.get()));
    }

    TRI_IF_FAILURE("SkiplistIndex::permutationEQ") {
      THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
    }

    TRI_IF_FAILURE("SkiplistIndex::permutationArrayIN") {
      THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
    }

    auto const& last = ops.back();
    for (auto const& op : last) {
      bool isReverseOrder = true;
      TRI_ASSERT(op->numMembers() == 2);

      auto value = op->getMember(0);
      if (value->isAttributeAccessForVariable(paramPair) && paramPair.first == var) {
        value = op->getMember(1);
        TRI_ASSERT(!(value->isAttributeAccessForVariable(paramPair) && paramPair.first == var));
        isReverseOrder = false;
      }
      switch (op->type) {
        case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_LT:
          if (isReverseOrder) {
            _includeLower = false;
          } else {
            _includeUpper = false;
          }
          // intentionally falls through
        case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_LE:
          if (isReverseOrder) {
            value->toVelocyPackValue(*(_lowerBuilder.get()));
          } else {
            value->toVelocyPackValue(*(_upperBuilder.get()));
          }
          break;
        case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_GT:
          if (isReverseOrder) {
            _includeUpper = false;
          } else {
            _includeLower = false;
          }
          // intentionally falls through
        case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_GE:
          if (isReverseOrder) {
            value->toVelocyPackValue(*(_upperBuilder.get()));
          } else {
            value->toVelocyPackValue(*(_lowerBuilder.get()));
          }
          break;
        default:
          TRI_ASSERT(false);
      }
    }
    _lowerBuilder->close();
    _lowerSlice = _lowerBuilder->slice();

    _upperBuilder->close();
    _upperSlice = _upperBuilder->slice();
  } else {
    for (auto const& oplist : ops) {
      TRI_ASSERT(oplist.size() == 1);
      auto const& op = oplist[0];
      TRI_ASSERT(op->type == arangodb::aql::NODE_TYPE_OPERATOR_BINARY_EQ ||
                 op->type == arangodb::aql::NODE_TYPE_OPERATOR_BINARY_IN);
      TRI_ASSERT(op->numMembers() == 2);
      auto value = op->getMember(0);
      if (value->isAttributeAccessForVariable(paramPair) && paramPair.first == var) {
        value = op->getMember(1);
        TRI_ASSERT(!(value->isAttributeAccessForVariable(paramPair) && paramPair.first == var));
      }
      value->toVelocyPackValue(*(_lowerBuilder.get()));
    }

    TRI_IF_FAILURE("SkiplistIndex::permutationEQ") {
      THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
    }

    TRI_IF_FAILURE("SkiplistIndex::permutationArrayIN") {
      THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
    }
    _lowerBuilder->close();
    _lowerSlice = _lowerBuilder->slice();
    _upperSlice = _lowerBuilder->slice();
  }
}

bool MMFilesSkiplistLookupBuilder::next() {
  // The first search value is created during creation.
  // So next is always false.
  return false;
}

MMFilesSkiplistInLookupBuilder::MMFilesSkiplistInLookupBuilder(
    transaction::Methods* trx,
    std::vector<std::vector<arangodb::aql::AstNode const*>>& ops,
    arangodb::aql::Variable const* var, bool reverse)
    : MMFilesBaseSkiplistLookupBuilder(trx), _dataBuilder(trx), _done(false) {
  TRI_ASSERT(!ops.empty());  // We certainly do not need IN here
  transaction::BuilderLeaser tmp(trx);
  std::set<VPackSlice, arangodb::basics::VelocyPackHelper::VPackSorted<true>> unique_set(
      (arangodb::basics::VelocyPackHelper::VPackSorted<true>(reverse)));
  std::pair<arangodb::aql::Variable const*, std::vector<arangodb::basics::AttributeName>> paramPair;

  _dataBuilder->clear();
  _dataBuilder->openArray();

  // The == and IN part
  for (size_t i = 0; i < ops.size() - 1; ++i) {
    auto const& oplist = ops[i];
    TRI_ASSERT(oplist.size() == 1);
    auto const& op = oplist[0];
    TRI_ASSERT(op->numMembers() == 2);
    auto value = op->getMember(0);
    bool valueLeft = true;
    if (value->isAttributeAccessForVariable(paramPair) && paramPair.first == var) {
      valueLeft = false;
      value = op->getMember(1);
      TRI_ASSERT(!(value->isAttributeAccessForVariable(paramPair) && paramPair.first == var));
    }
    if (op->type == arangodb::aql::NODE_TYPE_OPERATOR_BINARY_IN) {
      if (valueLeft) {
        // Case: value IN x.a
        // This is identical to == for the index.
        value->toVelocyPackValue(*(_dataBuilder.get()));
      } else {
        // Case: x.a IN value
        TRI_ASSERT(value->numMembers() > 0);
        tmp->clear();
        unique_set.clear();
        value->toVelocyPackValue(*(tmp.get()));
        for (VPackSlice it : VPackArrayIterator(tmp->slice())) {
          unique_set.emplace(it);
        }
        TRI_IF_FAILURE("SkiplistIndex::permutationIN") {
          THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
        }
        _inPositions.emplace_back(i, 0, unique_set.size());
        _dataBuilder->openArray();
        for (auto const& it : unique_set) {
          _dataBuilder->add(it);
        }
        _dataBuilder->close();
      }
    } else {
      TRI_ASSERT(op->type == arangodb::aql::NODE_TYPE_OPERATOR_BINARY_EQ);
      value->toVelocyPackValue(*(_dataBuilder.get()));
    }
  }
  auto const& last = ops.back();
  arangodb::aql::AstNode const* lower = nullptr;
  arangodb::aql::AstNode const* upper = nullptr;

  _isEquality = false;

  for (auto const& op : last) {
    bool isReverseOrder = true;
    TRI_ASSERT(op->numMembers() == 2);

    auto value = op->getMember(0);
    if (value->isAttributeAccessForVariable(paramPair) && paramPair.first == var) {
      value = op->getMember(1);
      TRI_ASSERT(!(value->isAttributeAccessForVariable(paramPair) && paramPair.first == var));
      isReverseOrder = false;
    }

    switch (op->type) {
      case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_LT:
        if (isReverseOrder) {
          _includeLower = false;
        } else {
          _includeUpper = false;
        }
        // intentionally falls through
      case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_LE:
        if (isReverseOrder) {
          TRI_ASSERT(lower == nullptr);
          lower = value;
        } else {
          TRI_ASSERT(upper == nullptr);
          upper = value;
        }
        break;
      case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_GT:
        if (isReverseOrder) {
          _includeUpper = false;
        } else {
          _includeLower = false;
        }
        // intentionally falls through
      case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_GE:
        if (isReverseOrder) {
          TRI_ASSERT(upper == nullptr);
          upper = value;
        } else {
          TRI_ASSERT(lower == nullptr);
          lower = value;
        }
        break;
      case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_IN:
        TRI_ASSERT(upper == nullptr);
        TRI_ASSERT(lower == nullptr);
        TRI_ASSERT(value->numMembers() > 0);
        tmp->clear();
        unique_set.clear();
        value->toVelocyPackValue(*(tmp.get()));
        for (auto const& it : VPackArrayIterator(tmp->slice())) {
          unique_set.emplace(it);
        }
        TRI_IF_FAILURE("Index::permutationIN") {
          THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
        }
        _inPositions.emplace_back(ops.size() - 1, 0, unique_set.size());
        _dataBuilder->openArray();
        for (auto const& it : unique_set) {
          _dataBuilder->add(it);
        }
        _dataBuilder->close();
        _isEquality = true;
        _dataBuilder->close();

        buildSearchValues();
        return;
      case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_EQ:
        TRI_ASSERT(upper == nullptr);
        TRI_ASSERT(lower == nullptr);
        value->toVelocyPackValue(*(_dataBuilder.get()));
        _isEquality = true;
        _dataBuilder->close();

        buildSearchValues();
        return;
      default:
        TRI_ASSERT(false);
    }
  }
  _dataBuilder->openArray();
  if (lower == nullptr) {
    _dataBuilder->add(arangodb::velocypack::Slice::nullSlice());
  } else {
    lower->toVelocyPackValue(*(_dataBuilder.get()));
  }

  if (upper == nullptr) {
    _dataBuilder->add(arangodb::velocypack::Slice::nullSlice());
  } else {
    upper->toVelocyPackValue(*(_dataBuilder.get()));
  }
  _dataBuilder->close();
  _dataBuilder->close();

  buildSearchValues();
}

bool MMFilesSkiplistInLookupBuilder::next() {
  if (_done || !forwardInPosition()) {
    return false;
  }
  buildSearchValues();
  return true;
}

bool MMFilesSkiplistInLookupBuilder::forwardInPosition() {
  std::list<PosStruct>::reverse_iterator it = _inPositions.rbegin();
  while (it != _inPositions.rend()) {
    it->current++;
    TRI_ASSERT(it->_max > 0);
    if (it->current < it->_max) {
      return true;
      // Okay we increased this, next search value;
    }
    it->current = 0;
    ++it;
  }
  _done = true;
  // If we get here all positions are reset to 0.
  // We are done, no further combination
  return false;
}

void MMFilesSkiplistInLookupBuilder::buildSearchValues() {
  auto inPos = _inPositions.begin();
  _lowerBuilder->clear();
  _lowerBuilder->openArray();

  VPackSlice data = _dataBuilder->slice();
  if (!_isEquality) {
    _upperBuilder->clear();
    _upperBuilder->openArray();

    size_t const n = data.length();

    for (size_t i = 0; i < n - 1; ++i) {
      if (inPos != _inPositions.end() && i == inPos->field) {
        VPackSlice s = data.at(i).at(inPos->current);
        _lowerBuilder->add(s);
        _upperBuilder->add(s);
        inPos++;
      } else {
        VPackSlice s = data.at(i);
        _lowerBuilder->add(s);
        _upperBuilder->add(s);
      }
    }

    VPackSlice bounds = data.at(n - 1);
    TRI_ASSERT(bounds.isArray());
    TRI_ASSERT(bounds.length() == 2);
    VPackSlice b = bounds.at(0);
    if (!b.isNull()) {
      _lowerBuilder->add(b);
    }
    _lowerBuilder->close();
    _lowerSlice = _lowerBuilder->slice();

    b = bounds.at(1);
    if (!b.isNull()) {
      _upperBuilder->add(b);
    }

    _upperBuilder->close();
    _upperSlice = _upperBuilder->slice();
  } else {
    size_t const n = data.length();

    for (size_t i = 0; i < n; ++i) {
      if (inPos != _inPositions.end() && i == inPos->field) {
        _lowerBuilder->add(data.at(i).at(inPos->current));
        inPos++;
      } else {
        _lowerBuilder->add(data.at(i));
      }
    }
    _lowerBuilder->close();
    _lowerSlice = _lowerBuilder->slice();
    _upperSlice = _lowerBuilder->slice();
  }
}

MMFilesSkiplistIterator::MMFilesSkiplistIterator(
    LogicalCollection* collection, transaction::Methods* trx,
    arangodb::MMFilesSkiplistIndex const* index,
    TRI_Skiplist const* skiplist, size_t numPaths,
    std::function<int(void*, MMFilesSkiplistIndexElement const*,
                      MMFilesSkiplistIndexElement const*, MMFilesSkiplistCmpType)> const& CmpElmElm,
    bool reverse, MMFilesBaseSkiplistLookupBuilder* builder)
    : IndexIterator(collection, trx),
      _skiplistIndex(skiplist),
      _context(collection, nullptr, index->fields().size()),
      _numPaths(numPaths),
      _reverse(reverse),
      _cursor(nullptr),
      _currentInterval(0),
      _builder(builder),
      _CmpElmElm(CmpElmElm) {
  TRI_ASSERT(_builder != nullptr);
  initNextInterval();  // Initializes the cursor
  TRI_ASSERT((_intervals.empty() && _cursor == nullptr) ||
             (!_intervals.empty() && _cursor != nullptr));
}

/// @brief Checks if the interval is valid. It is declared invalid if
///        one border is nullptr or the right is lower than left.
bool MMFilesSkiplistIterator::intervalValid(void* userData, Node* left, Node* right) const {
  if (left == nullptr) {
    return false;
  }
  if (right == nullptr) {
    return false;
  }
  if (left == right) {
    // Exactly one result. Improve speed on unique indexes
    return true;
  }
  if (_CmpElmElm(userData, left->document(), right->document(),
                 arangodb::SKIPLIST_CMP_TOTORDER) > 0) {
    return false;
  }
  return true;
}

/// @brief Reset the cursor
void MMFilesSkiplistIterator::reset() {
  // If _intervals is empty at this point
  // the cursor does not contain any
  // document at all. Reset is pointless
  if (!_intervals.empty()) {
    // We reset to the first interval and reset the cursor
    _currentInterval = 0;
    if (_reverse) {
      _cursor = _intervals[0].second;
    } else {
      _cursor = _intervals[0].first;
    }
  }
}

bool MMFilesSkiplistIterator::next(LocalDocumentIdCallback const& cb, size_t limit) {
  while (limit > 0) {
    if (_cursor == nullptr) {
      // We are exhausted already, sorry
      return false;
    }
    TRI_ASSERT(_currentInterval < _intervals.size());
    auto const& interval = _intervals[_currentInterval];
    Node* tmp = _cursor;
    if (_reverse) {
      if (_cursor == interval.first) {
        forwardCursor();
      } else {
        _cursor = _cursor->prevNode();
      }
    } else {
      if (_cursor == interval.second) {
        forwardCursor();
      } else {
        _cursor = _cursor->nextNode();
      }
    }
    TRI_ASSERT(tmp != nullptr);
    TRI_ASSERT(tmp->document() != nullptr);

    cb(tmp->document()->localDocumentId());
    limit--;
  }
  return true;
}

bool MMFilesSkiplistIterator::nextDocument(DocumentCallback const& cb, size_t limit) {
  _documentIds.clear();
  _documentIds.reserve(limit);

  bool done = false;
  while (limit > 0) {
    if (_cursor == nullptr) {
      // We are exhausted already, sorry
      done = true;
      break;
    }
    TRI_ASSERT(_currentInterval < _intervals.size());
    auto const& interval = _intervals[_currentInterval];
    Node* tmp = _cursor;
    if (_reverse) {
      if (_cursor == interval.first) {
        forwardCursor();
      } else {
        _cursor = _cursor->prevNode();
      }
    } else {
      if (_cursor == interval.second) {
        forwardCursor();
      } else {
        _cursor = _cursor->nextNode();
      }
    }
    TRI_ASSERT(tmp != nullptr);
    TRI_ASSERT(tmp->document() != nullptr);

    _documentIds.emplace_back(std::make_pair(tmp->document()->localDocumentId(), nullptr));
    limit--;
  }

  auto physical = static_cast<MMFilesCollection*>(_collection->getPhysical());
  physical->readDocumentWithCallback(_trx, _documentIds, cb);
  return !done;
}

void MMFilesSkiplistIterator::forwardCursor() {
  _currentInterval++;
  if (_currentInterval < _intervals.size()) {
    auto const& interval = _intervals[_currentInterval];
    if (_reverse) {
      _cursor = interval.second;
    } else {
      _cursor = interval.first;
    }
    return;
  }
  _cursor = nullptr;
  if (_builder->next()) {
    initNextInterval();
  }
}

void MMFilesSkiplistIterator::initNextInterval() {
  // We will always point the cursor to the resulting interval if any.
  // We do not take responsibility for the Nodes!
  Node* rightBorder = nullptr;
  Node* leftBorder = nullptr;

  while (true) {
    if (_builder->isEquality()) {
      rightBorder =
          _skiplistIndex->rightKeyLookup(&_context, _builder->getLowerLookup());
      if (rightBorder == _skiplistIndex->startNode()) {
        // No matching elements. Next interval
        if (!_builder->next()) {
          // No next interval. We are done.
          return;
        }
        // Builder moved forward. Try again.
        continue;
      }
      leftBorder = _skiplistIndex->leftKeyLookup(&_context, _builder->getLowerLookup());
      leftBorder = leftBorder->nextNode();
      // NOTE: rightBorder < leftBorder => no Match.
      // Will be checked by interval valid
    } else {
      if (_builder->includeLower()) {
        leftBorder =
            _skiplistIndex->leftKeyLookup(&_context, _builder->getLowerLookup());
        // leftKeyLookup guarantees that we find the element before search.
      } else {
        leftBorder =
            _skiplistIndex->rightKeyLookup(&_context, _builder->getLowerLookup());
        // leftBorder is identical or smaller than search
      }
      // This is the first element not to be returned, but the next one
      // Also save for the startNode, it should never be contained in the index.
      leftBorder = leftBorder->nextNode();

      if (_builder->includeUpper()) {
        rightBorder =
            _skiplistIndex->rightKeyLookup(&_context, _builder->getUpperLookup());
      } else {
        rightBorder =
            _skiplistIndex->leftKeyLookup(&_context, _builder->getUpperLookup());
      }
      if (rightBorder == _skiplistIndex->startNode()) {
        // No match make interval invalid
        rightBorder = nullptr;
      }
      // else rightBorder is correct
    }
    if (!intervalValid(&_context, leftBorder, rightBorder)) {
      // No matching elements. Next interval
      if (!_builder->next()) {
        // No next interval. We are done.
        return;
      }
      // Builder moved forward. Try again.
      continue;
    }
    TRI_ASSERT(_currentInterval == _intervals.size());
    _intervals.emplace_back(leftBorder, rightBorder);
    if (_reverse) {
      _cursor = rightBorder;
    } else {
      _cursor = leftBorder;
    }
    // Next valid interal initialized. Return;
    return;
  }
}

/// @brief create the skiplist index
MMFilesSkiplistIndex::MMFilesSkiplistIndex(TRI_idx_iid_t iid,
                                           arangodb::LogicalCollection& collection,
                                           arangodb::velocypack::Slice const& info)
    : MMFilesPathBasedIndex(iid, collection, info, sizeof(LocalDocumentId), true),
      CmpElmElm(this),
      CmpKeyElm(this),
      _skiplistIndex(nullptr) {
  _skiplistIndex = new TRI_Skiplist(CmpElmElm, CmpKeyElm,
                                    [this](MMFilesSkiplistIndexElement* element) {
                                      _allocator->deallocate(element);
                                    },
                                    _unique, _useExpansion);
}

/// @brief destroy the skiplist index
MMFilesSkiplistIndex::~MMFilesSkiplistIndex() { delete _skiplistIndex; }

size_t MMFilesSkiplistIndex::memory() const {
  return _skiplistIndex->memoryUsage() +
         static_cast<size_t>(_skiplistIndex->getNrUsed()) *
             MMFilesSkiplistIndexElement::baseMemoryUsage(_paths.size());
}

/// @brief return a VelocyPack representation of the index figures
void MMFilesSkiplistIndex::toVelocyPackFigures(VPackBuilder& builder) const {
  MMFilesPathBasedIndex::toVelocyPackFigures(builder);
  _skiplistIndex->appendToVelocyPack(builder);
}

/// @brief inserts a document into a skiplist index
Result MMFilesSkiplistIndex::insert(transaction::Methods& trx,
                                    LocalDocumentId const& documentId,
                                    velocypack::Slice const& doc,
                                    Index::OperationMode mode) {
  std::vector<MMFilesSkiplistIndexElement*> elements;
  Result res;

  int r;

  try {
    r = fillElement<MMFilesSkiplistIndexElement>(elements, documentId, doc);
  } catch (basics::Exception const& ex) {
    r = ex.code();
  } catch (std::bad_alloc const&) {
    r = TRI_ERROR_OUT_OF_MEMORY;
  } catch (...) {
    r = TRI_ERROR_INTERNAL;
  }

  if (r != TRI_ERROR_NO_ERROR) {
    for (auto& element : elements) {
      // free all elements to prevent leak
      _allocator->deallocate(element);
    }
    return addErrorMsg(res, r);
  }

  MMFilesIndexLookupContext context(&_collection, nullptr, numPaths());

  // insert into the index. the memory for the element will be owned or freed
  // by the index
  size_t const count = elements.size();

  size_t badIndex = 0;
  for (size_t i = 0; i < count; ++i) {
    r = _skiplistIndex->insert(&context, elements[i]);

    if (r != TRI_ERROR_NO_ERROR) {
      badIndex = i;

      // Note: this element is freed already
      for (size_t j = i; j < count; ++j) {
        _allocator->deallocate(elements[j]);
      }
      for (size_t j = 0; j < i; ++j) {
        _skiplistIndex->remove(&context, elements[j]);
        // No need to free elements[j] skiplist has taken over already
      }

      if (r == TRI_ERROR_ARANGO_UNIQUE_CONSTRAINT_VIOLATED && !_unique) {
        // We ignore unique_constraint violated if we are not unique
        r = TRI_ERROR_NO_ERROR;
      }

      break;
    }
  }

  if (r == TRI_ERROR_ARANGO_UNIQUE_CONSTRAINT_VIOLATED) {
    elements.clear();

    // need to rebuild elements, find conflicting key to return error,
    // and then free elements again
    int innerRes = TRI_ERROR_NO_ERROR;
    try {
      innerRes = fillElement<MMFilesSkiplistIndexElement>(elements, documentId, doc);
    } catch (basics::Exception const& ex) {
      innerRes = ex.code();
    } catch (std::bad_alloc const&) {
      innerRes = TRI_ERROR_OUT_OF_MEMORY;
    } catch (...) {
      innerRes = TRI_ERROR_INTERNAL;
    }

    auto guard = scopeGuard([this, &elements] {
      for (auto& element : elements) {
        // free all elements to prevent leak
        _allocator->deallocate(element);
      }
    });

    if (innerRes != TRI_ERROR_NO_ERROR) {
      return addErrorMsg(res, innerRes);
    }

    auto found = _skiplistIndex->rightLookup(&context, elements[badIndex]);
    TRI_ASSERT(found);
    LocalDocumentId rev(found->document()->localDocumentId());
    std::string existingId;

    _collection.getPhysical()->readDocumentWithCallback(
        &trx, rev, [&existingId](LocalDocumentId const&, velocypack::Slice doc) -> void {
          existingId = doc.get(StaticStrings::KeyString).copyString();
        });

    if (mode == OperationMode::internal) {
      return res.reset(r, std::move(existingId));
    }

    return addErrorMsg(res, r, existingId);
  }

  return addErrorMsg(res, r);
}

/// @brief removes a document from a skiplist index
Result MMFilesSkiplistIndex::remove(transaction::Methods& trx,
                                    LocalDocumentId const& documentId,
                                    velocypack::Slice const& doc,
                                    Index::OperationMode mode) {
  std::vector<MMFilesSkiplistIndexElement*> elements;
  Result res;
  int r;

  try {
    r = fillElement<MMFilesSkiplistIndexElement>(elements, documentId, doc);
  } catch (basics::Exception const& ex) {
    r = ex.code();
  } catch (std::bad_alloc const&) {
    r = TRI_ERROR_OUT_OF_MEMORY;
  } catch (...) {
    r = TRI_ERROR_INTERNAL;
  }

  if (r != TRI_ERROR_NO_ERROR) {
    for (auto& element : elements) {
      // free all elements to prevent leak
      _allocator->deallocate(element);
    }
    return addErrorMsg(res, r);
  }

  MMFilesIndexLookupContext context(&_collection, nullptr, numPaths());

  // attempt the removal for skiplist indexes
  // ownership for the index element is transferred to the index
  size_t const count = elements.size();

  for (size_t i = 0; i < count; ++i) {
    int result = _skiplistIndex->remove(&context, elements[i]);

    // we may be looping through this multiple times, and if an error
    // occurs, we want to keep it
    if (result != TRI_ERROR_NO_ERROR) {
      r = result;
    }

    _allocator->deallocate(elements[i]);
  }

  return addErrorMsg(res, r);
}

void MMFilesSkiplistIndex::unload() { _skiplistIndex->truncate(true); }

/// @brief Checks if the interval is valid. It is declared invalid if
///        one border is nullptr or the right is lower than left.
bool MMFilesSkiplistIndex::intervalValid(void* userData, Node* left, Node* right) const {
  if (left == nullptr) {
    return false;
  }
  if (right == nullptr) {
    return false;
  }
  if (left == right) {
    // Exactly one result. Improve speed on unique indexes
    return true;
  }
  if (CmpElmElm(userData, left->document(), right->document(),
                arangodb::SKIPLIST_CMP_TOTORDER) > 0) {
    return false;
  }
  return true;
}

/// @brief compares a key with an element in a skip list, generic callback
int MMFilesSkiplistIndex::KeyElementComparator::operator()(
    void* userData, VPackSlice const* leftKey,
    MMFilesSkiplistIndexElement const* rightElement) const {
  TRI_ASSERT(nullptr != leftKey);
  TRI_ASSERT(nullptr != rightElement);

  // Note that the key might contain fewer fields than there are indexed
  // attributes, therefore we only run the following loop to
  // leftKey->_numFields.
  TRI_ASSERT(leftKey->isArray());
  size_t numFields = leftKey->length();
  for (size_t j = 0; j < numFields; j++) {
    VPackSlice field = leftKey->at(j);
    int compareResult = CompareKeyElement(userData, &field, rightElement, j);
    if (compareResult != 0) {
      return compareResult;
    }
  }

  return 0;
}

/// @brief compares two elements in a skip list, this is the generic callback
int MMFilesSkiplistIndex::ElementElementComparator::operator()(
    void* userData, MMFilesSkiplistIndexElement const* leftElement,
    MMFilesSkiplistIndexElement const* rightElement, MMFilesSkiplistCmpType cmptype) const {
  TRI_ASSERT(nullptr != leftElement);
  TRI_ASSERT(nullptr != rightElement);

  // ..........................................................................
  // The document could be the same -- so no further comparison is required.
  // ..........................................................................

  if (leftElement == rightElement ||
      (!_idx->_skiplistIndex->isArray() &&
       leftElement->localDocumentId() == rightElement->localDocumentId())) {
    return 0;
  }

  for (size_t j = 0; j < _idx->numPaths(); j++) {
    int compareResult = CompareElementElement(userData, leftElement, j, rightElement, j);

    if (compareResult != 0) {
      return compareResult;
    }
  }

  // ...........................................................................
  // This is where the difference between the preorder and the proper total
  // order comes into play. Here if the 'keys' are the same,
  // but the doc ptr is different (which it is since we are here), then
  // we return 0 if we use the preorder and look at the _key attribute
  // otherwise.
  // ...........................................................................

  if (arangodb::SKIPLIST_CMP_PREORDER == cmptype) {
    return 0;
  }

  // We break this tie in the key comparison by looking at the key:
  if (leftElement->localDocumentId() < rightElement->localDocumentId()) {
    return -1;
  }
  if (leftElement->localDocumentId() > rightElement->localDocumentId()) {
    return 1;
  }
  return 0;
}

bool MMFilesSkiplistIndex::accessFitsIndex(
    arangodb::aql::AstNode const* access, arangodb::aql::AstNode const* other,
    arangodb::aql::AstNode const* op, arangodb::aql::Variable const* reference,
    std::vector<std::vector<arangodb::aql::AstNode const*>>& found,
    std::unordered_set<std::string>& nonNullAttributes) const {
  if (!this->canUseConditionPart(access, other, op, reference, nonNullAttributes, true)) {
    return false;
  }

  arangodb::aql::AstNode const* what = access;
  std::pair<arangodb::aql::Variable const*, std::vector<arangodb::basics::AttributeName>> attributeData;

  if (op->type != arangodb::aql::NODE_TYPE_OPERATOR_BINARY_IN) {
    if (!what->isAttributeAccessForVariable(attributeData) || attributeData.first != reference) {
      // this access is not referencing this collection
      return false;
    }
    if (arangodb::basics::TRI_AttributeNamesHaveExpansion(attributeData.second)) {
      // doc.value[*] == 'value'
      return false;
    }
    if (isAttributeExpanded(attributeData.second)) {
      // doc.value == 'value' (with an array index)
      return false;
    }
  } else {
    // ok, we do have an IN here... check if it's something like 'value' IN
    // doc.value[*]
    TRI_ASSERT(op->type == arangodb::aql::NODE_TYPE_OPERATOR_BINARY_IN);
    bool canUse = false;

    if (what->isAttributeAccessForVariable(attributeData) && attributeData.first == reference &&
        !arangodb::basics::TRI_AttributeNamesHaveExpansion(attributeData.second) &&
        attributeMatches(attributeData.second)) {
      // doc.value IN 'value'
      // can use this index
      canUse = true;
    } else {
      // check for  'value' IN doc.value  AND  'value' IN doc.value[*]
      what = other;
      if (what->isAttributeAccessForVariable(attributeData) &&
          attributeData.first == reference && isAttributeExpanded(attributeData.second) &&
          attributeMatches(attributeData.second)) {
        canUse = true;
      }
    }

    if (!canUse) {
      return false;
    }
  }

  std::vector<arangodb::basics::AttributeName> const& fieldNames = attributeData.second;

  for (size_t i = 0; i < _fields.size(); ++i) {
    if (_fields[i].size() != fieldNames.size()) {
      // attribute path length differs
      continue;
    }

    if (this->isAttributeExpanded(i) && op->type != arangodb::aql::NODE_TYPE_OPERATOR_BINARY_IN) {
      // If this attribute is correct or not, it could only serve for IN
      continue;
    }

    bool match = arangodb::basics::AttributeName::isIdentical(_fields[i], fieldNames, true);

    if (match) {
      // mark ith attribute as being covered
      found[i].emplace_back(op);

      TRI_IF_FAILURE("SkiplistIndex::accessFitsIndex") {
        THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
      }

      return true;
    }
  }

  return false;
}

bool MMFilesSkiplistIndex::findMatchingConditions(
    arangodb::aql::AstNode const* node, arangodb::aql::Variable const* reference,
    std::vector<std::vector<arangodb::aql::AstNode const*>>& mapping, bool& usesIn) const {
  std::unordered_set<std::string> nonNullAttributes;
  usesIn = false;

  for (size_t i = 0; i < node->numMembers(); ++i) {
    auto op = node->getMember(i);

    switch (op->type) {
      case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_EQ:
      case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_LT:
      case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_LE:
      case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_GT:
      case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_GE: {
        TRI_ASSERT(op->numMembers() == 2);
        accessFitsIndex(op->getMember(0), op->getMember(1), op, reference,
                        mapping, nonNullAttributes);
        accessFitsIndex(op->getMember(1), op->getMember(0), op, reference,
                        mapping, nonNullAttributes);
        break;
      }
      case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_IN: {
        auto m = op->getMember(1);
        if (accessFitsIndex(op->getMember(0), m, op, reference, mapping, nonNullAttributes)) {
          if (m->numMembers() == 0) {
            // We want to do an IN [].
            // No results
            // Even if we cannot use the index.
            return false;
          }
        }
        break;
      }

      default: {
        TRI_ASSERT(false);
        break;
      }
    }
  }

  for (size_t i = 0; i < mapping.size(); ++i) {
    auto const& conditions = mapping[i];
    if (conditions.empty()) {
      // We do not have any condition for this field.
      // Remove it and everything afterwards.
      mapping.resize(i);
      TRI_ASSERT(i == mapping.size());
      break;
    }
    TRI_ASSERT(conditions.size() <= 2);
    auto const& first = conditions[0];
    switch (first->type) {
      case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_IN:
        if (first->getMember(1)->isArray()) {
          usesIn = true;
        }
        // intentionally falls through
      case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_EQ:
        TRI_ASSERT(conditions.size() == 1);
        break;

      default: {
        // All conditions after this cannot be used.
        // shrink and break outer for loop
        mapping.resize(i + 1);
        TRI_ASSERT(i + 1 == mapping.size());
        return true;
      }
    }
  }
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
  for (auto const& it : mapping) {
    TRI_ASSERT(!it.empty());
  }
#endif

  return true;
}

std::unique_ptr<IndexIterator> MMFilesSkiplistIndex::iteratorForCondition(
    transaction::Methods* trx,
    arangodb::aql::AstNode const* node,
    arangodb::aql::Variable const* reference, IndexIteratorOptions const& opts) {
  TRI_ASSERT(!isSorted() || opts.sorted);
  std::vector<std::vector<arangodb::aql::AstNode const*>> mapping;
  bool usesIn = false;
  if (node != nullptr) {
    mapping.resize(_fields.size());  // We use the default constructor. Mapping
                                     // will have _fields many entries.
    TRI_ASSERT(mapping.size() == _fields.size());
    if (!findMatchingConditions(node, reference, mapping, usesIn)) {
      return std::make_unique<EmptyIndexIterator>(&_collection, trx);
    }
  } else {
    TRI_IF_FAILURE("SkiplistIndex::noSortIterator") {
      THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
    }
  }

  TRI_IF_FAILURE("SkiplistIndex::noIterator") {
    THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
  }

  if (usesIn) {
    auto builder = std::make_unique<MMFilesSkiplistInLookupBuilder>(trx, mapping, reference,
                                                                    !opts.ascending);

    return std::make_unique<MMFilesSkiplistIterator>(&_collection, trx, this,
                                       _skiplistIndex, numPaths(), CmpElmElm,
                                       !opts.ascending, builder.release());
  }

  auto builder = std::make_unique<MMFilesSkiplistLookupBuilder>(trx, mapping, reference,
                                                                !opts.ascending);

  return std::make_unique<MMFilesSkiplistIterator>(&_collection, trx, this,
                                     _skiplistIndex, numPaths(), CmpElmElm,
                                     !opts.ascending, builder.release());
}

Index::UsageCosts MMFilesSkiplistIndex::supportsFilterCondition(
    std::vector<std::shared_ptr<arangodb::Index>> const& allIndexes,
    arangodb::aql::AstNode const* node, arangodb::aql::Variable const* reference,
    size_t itemsInIndex) const {
  return SortedIndexAttributeMatcher::supportsFilterCondition(allIndexes, this,
                                                                node, reference,
                                                                itemsInIndex);
}

Index::UsageCosts MMFilesSkiplistIndex::supportsSortCondition(arangodb::aql::SortCondition const* sortCondition,
                                                              arangodb::aql::Variable const* reference,
                                                              size_t itemsInIndex) const {
  return SortedIndexAttributeMatcher::supportsSortCondition(this, sortCondition, reference, itemsInIndex);
}

/// @brief specializes the condition for use with the index
arangodb::aql::AstNode* MMFilesSkiplistIndex::specializeCondition(
    arangodb::aql::AstNode* node, arangodb::aql::Variable const* reference) const {
  return SortedIndexAttributeMatcher::specializeCondition(this, node, reference);
}