////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Max Neunhoeffer
////////////////////////////////////////////////////////////////////////////////

#include "Aql/Optimizer.h"
#include "Aql/ExecutionEngine.h"
#include "Aql/OptimizerRules.h"
#include "Cluster/ServerState.h"

using namespace arangodb::aql;

arangodb::Mutex Optimizer::SetupLock;

////////////////////////////////////////////////////////////////////////////////
// @brief list of all rules
////////////////////////////////////////////////////////////////////////////////

std::map<int, Optimizer::Rule> Optimizer::_rules;

////////////////////////////////////////////////////////////////////////////////
// @brief lookup from rule name to rule level
////////////////////////////////////////////////////////////////////////////////

std::unordered_map<std::string, int> Optimizer::_ruleLookup;

////////////////////////////////////////////////////////////////////////////////
// @brief constructor, this will initialize the rules database
////////////////////////////////////////////////////////////////////////////////

Optimizer::Optimizer(size_t maxNumberOfPlans)
    : _maxNumberOfPlans(maxNumberOfPlans > 0 ? maxNumberOfPlans
                                             : DefaultMaxNumberOfPlans) {
  if (_rules.empty()) {
    setupRules();
  }
}

////////////////////////////////////////////////////////////////////////////////
// @brief add a plan to the optimizer
////////////////////////////////////////////////////////////////////////////////

bool Optimizer::addPlan(ExecutionPlan* plan, Rule const* rule, bool wasModified,
                        int newLevel) {
  TRI_ASSERT(plan != nullptr);

  if (newLevel > 0) {
    // use user-specified new level
    _newPlans.push_back(plan, newLevel);
  } else {
    // use rule's level
    _newPlans.push_back(plan, rule->level);
  }

  if (wasModified) {
    if (!rule->isHidden) {
      // register which rules modified / created the plan
      // hidden rules are excluded here
      plan->addAppliedRule(static_cast<int>(rule->level));
    }

    plan->clearVarUsageComputed();
    plan->invalidateCost();

    plan->findVarUsage();
  }

  if (_newPlans.size() >= _maxNumberOfPlans) {
    return false;
  }

  return true;
}

////////////////////////////////////////////////////////////////////////////////
// @brief the actual optimization
////////////////////////////////////////////////////////////////////////////////

int Optimizer::createPlans(ExecutionPlan* plan,
                           std::vector<std::string> const& rulesSpecification,
                           bool inspectSimplePlans) {
  if (!inspectSimplePlans &&
      !arangodb::ServerState::instance()->isCoordinator() &&
      plan->isDeadSimple()) {
    // the plan is so simple that any further optimizations would probably cost
    // more than simply executing the plan
    _plans.clear();
    try {
      _plans.push_back(plan, 0);
    } catch (...) {
      delete plan;
      throw;
    }

    estimatePlans();

    return TRI_ERROR_NO_ERROR;
  }

  bool runOnlyRequiredRules = false;
  int leastDoneLevel = 0;

  TRI_ASSERT(!_rules.empty());
  int maxRuleLevel = _rules.rbegin()->first;

  // which optimizer rules are disabled?
  std::unordered_set<int> disabledIds(getDisabledRuleIds(rulesSpecification));

  // _plans contains the previous optimisation result
  _plans.clear();
  try {
    _plans.push_back(plan, 0);
  } catch (...) {
    delete plan;
    throw;
  }

  _newPlans.clear();

  while (leastDoneLevel < maxRuleLevel) {
    // std::cout << "Have " << _plans.size() << " plans:" << std::endl;
    /*
    for (auto const& p : _plans.list) {
      p->show();
      std::cout << std::endl;
    }
    */

    // int count = 0;

    // For all current plans:
    while (_plans.size() > 0) {
      int level;
      auto p = _plans.pop_front(level);

      if (level >= maxRuleLevel) {
        _newPlans.push_back(p, level);  // nothing to do, just keep it
      } else {                          // find next rule
        auto it = _rules.upper_bound(level);
        TRI_ASSERT(it != _rules.end());

        /*
        std::cout << "Trying rule " << it->second.name << " with level "
                  << it->first << " on plan " << count++
                  << std::endl;
        */

        level = (*it).first;
        auto& rule = (*it).second;

        if ((runOnlyRequiredRules ||
             disabledIds.find(level) != disabledIds.end()) &&
            rule.canBeDisabled) {
          // we picked a disabled rule or we have reached the max number of
          // plans
          // and just skip this rule

          _newPlans.push_back(p, level);  // nothing to do, just keep it

          if (!rule.isHidden) {
            ++_stats.rulesSkipped;
          }
          // now try next
          continue;
        }

        try {
          TRI_IF_FAILURE("Optimizer::createPlansOom") {
            THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
          }

          if (!p->varUsageComputed()) {
            p->findVarUsage();
          }

          // all optimizer rule functions must obey the following guidelines:
          // - the original plan passed to the rule function must be deleted if
          // and only
          //   if it has not been added (back) to the optimizer (using addPlan).
          // - if the rule throws, then the original plan will be deleted by the
          // optimizer.
          //   thus the rule must not have deleted the plan itself or add it
          //   back to the
          //   optimizer
          rule.func(this, p, &rule);

          if (!rule.isHidden) {
            ++_stats.rulesExecuted;
          }
        } catch (...) {
          if (!_newPlans.isContained(p)) {
            // only delete the plan if not yet contained in _newPlans
            delete p;
          }
          throw;
        }
      }

      // future optimization: abort early here if we found a good-enough plan
      // a good-enough plan is probably every plan with costs below some
      // defined threshold. this requires plan costs to be calculated here
    }

    _plans.steal(_newPlans);
    leastDoneLevel = maxRuleLevel;
    for (auto const& l : _plans.levelDone) {
      if (l < leastDoneLevel) {
        leastDoneLevel = l;
      }
    }
    // std::cout << "Least done level is " << leastDoneLevel << std::endl;

    // Stop if the result gets out of hand:
    if (!runOnlyRequiredRules && _plans.size() >= _maxNumberOfPlans) {
      // must still iterate over all REQUIRED remaining transformation rules
      // because there are some rules which are required to make the query
      // work in cluster mode etc
      runOnlyRequiredRules = true;
    }
  }

  _stats.plansCreated = _plans.size();

  TRI_ASSERT(_plans.size() >= 1);

  estimatePlans();
  sortPlans();
#if 0
  // Only for debugging:
  std::cout << "Optimisation ends with " << _plans.size() << " plans."
            << std::endl;
  for (auto const& p : _plans.list) {
    p->show();
    std::cout << "costing: " << p->getCost() << std::endl;
    std::cout << std::endl;
  }
#endif

  return TRI_ERROR_NO_ERROR;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief translate a list of rule ids into rule names
////////////////////////////////////////////////////////////////////////////////

std::vector<std::string> Optimizer::translateRules(
    std::vector<int> const& rules) {
  std::vector<std::string> names;
  names.reserve(rules.size());

  for (auto const& rule : rules) {
    char const* name = translateRule(rule);

    if (name != nullptr) {
      names.emplace_back(std::string(name));
    }
  }
  return names;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief translate a single rule
////////////////////////////////////////////////////////////////////////////////

char const* Optimizer::translateRule(int rule) {
  auto it = _rules.find(rule);

  if (it != _rules.end()) {
    return (*it).second.name.c_str();
  }

  return nullptr;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief estimatePlans
////////////////////////////////////////////////////////////////////////////////

void Optimizer::estimatePlans() {
  for (auto& p : _plans.list) {
    if (!p->varUsageComputed()) {
      p->findVarUsage();
    }
    p->getCost();
    // this value is cached in the plan, so formally this step is
    // unnecessary, but for the sake of cleanliness...
  }
}

////////////////////////////////////////////////////////////////////////////////
/// @brief sortPlans
////////////////////////////////////////////////////////////////////////////////

void Optimizer::sortPlans() {
  std::sort(_plans.list.begin(), _plans.list.end(),
            [](ExecutionPlan* const& a, ExecutionPlan* const& b)
                -> bool { return a->getCost() < b->getCost(); });
}

////////////////////////////////////////////////////////////////////////////////
/// @brief look up the ids of all disabled rules
////////////////////////////////////////////////////////////////////////////////

std::unordered_set<int> Optimizer::getDisabledRuleIds(
    std::vector<std::string> const& names) const {
  std::unordered_set<int> disabled;

  // lookup ids of all disabled rules
  for (auto const& name : names) {
    if (name[0] == '-') {
      // disable rule
      if (name == "-all") {
        // disable all rules
        for (auto const& it : _rules) {
          disabled.emplace(it.first);
        }
      } else {
        // disable a specific rule
        auto it = _ruleLookup.find(std::string(name.c_str() + 1));

        if (it != _ruleLookup.end()) {
          disabled.emplace((*it).second);
        }
      }
    } else if (name[0] == '+') {
      // enable rule
      if (name == "+all") {
        // enable all rules
        disabled.clear();
      } else {
        auto it = _ruleLookup.find(std::string(name.c_str() + 1));

        if (it != _ruleLookup.end()) {
          disabled.erase((*it).second);
        }
      }
    }
  }

  return disabled;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief set up the optimizer rules once and forever
////////////////////////////////////////////////////////////////////////////////

void Optimizer::setupRules() {
  MUTEX_LOCKER(mutexLocker, SetupLock);

  if (!_rules.empty()) {
    // race condition... prevent duplicate registration of rules
    return;
  }

// List all the rules in the system here:
// lower level values mean earlier rule execution

// note that levels must be unique

//////////////////////////////////////////////////////////////////////////////
// "Pass 1": moving nodes "up" (potentially outside loops):
//////////////////////////////////////////////////////////////////////////////

#if 0
  // rule not yet tested
  registerRule("split-filters",
               splitFiltersRule,
               splitFiltersRule_pass1, 
               true);
#endif

  // determine the "right" type of CollectNode and
  // add a sort node for each COLLECT (may be removed later)
  // this rule cannot be turned off (otherwise, the query result might be
  // wrong!)
  registerHiddenRule("specialize-collect", specializeCollectRule,
                     specializeCollectRule_pass1, false);

  // move calculations up the dependency chain (to pull them out of
  // inner loops etc.)
  registerRule("move-calculations-up", moveCalculationsUpRule,
               moveCalculationsUpRule_pass1, true);

  // move filters up the dependency chain (to make result sets as small
  // as possible as early as possible)
  registerRule("move-filters-up", moveFiltersUpRule, moveFiltersUpRule_pass1,
               true);

  // remove redundant calculations
  registerRule("remove-redundant-calculations", removeRedundantCalculationsRule,
               removeRedundantCalculationsRule_pass1, true);

  //////////////////////////////////////////////////////////////////////////////
  /// "Pass 2": try to remove redundant or unnecessary nodes
  //////////////////////////////////////////////////////////////////////////////

  // remove filters from the query that are not necessary at all
  // filters that are always true will be removed entirely
  // filters that are always false will be replaced with a NoResults node
  registerRule("remove-unnecessary-filters", removeUnnecessaryFiltersRule,
               removeUnnecessaryFiltersRule_pass2, true);

  // remove calculations that are never necessary
  registerRule("remove-unnecessary-calculations",
               removeUnnecessaryCalculationsRule,
               removeUnnecessaryCalculationsRule_pass2, true);

  // remove redundant sort blocks
  registerRule("remove-redundant-sorts", removeRedundantSortsRule,
               removeRedundantSortsRule_pass2, true);

  //////////////////////////////////////////////////////////////////////////////
  /// "Pass 3": interchange EnumerateCollection nodes in all possible ways
  ///           this is level 500, please never let new plans from higher
  ///           levels go back to this or lower levels!
  //////////////////////////////////////////////////////////////////////////////

  registerRule("interchange-adjacent-enumerations",
               interchangeAdjacentEnumerationsRule,
               interchangeAdjacentEnumerationsRule_pass3, true);

  //////////////////////////////////////////////////////////////////////////////
  // "Pass 4": moving nodes "up" (potentially outside loops) (second try):
  //////////////////////////////////////////////////////////////////////////////

  // move calculations up the dependency chain (to pull them out of
  // inner loops etc.)
  registerRule("move-calculations-up-2", moveCalculationsUpRule,
               moveCalculationsUpRule_pass4, true);

  // move filters up the dependency chain (to make result sets as small
  // as possible as early as possible)
  registerRule("move-filters-up-2", moveFiltersUpRule, moveFiltersUpRule_pass4,
               true);

  // merge filters into traversals
  registerRule("merge-traversal-filter", mergeFilterIntoTraversalRule,
               mergeFilterIntoTraversalRule_pass6, true);

  //////////////////////////////////////////////////////////////////////////////
  /// "Pass 5": try to remove redundant or unnecessary nodes (second try)
  //////////////////////////////////////////////////////////////////////////////

  // remove filters from the query that are not necessary at all
  // filters that are always true will be removed entirely
  // filters that are always false will be replaced with a NoResults node
  registerRule("remove-unnecessary-filters-2", removeUnnecessaryFiltersRule,
               removeUnnecessaryFiltersRule_pass6, true);

  // remove redundant sort node
  registerRule("remove-redundant-sorts-2", removeRedundantSortsRule,
               removeRedundantSortsRule_pass5, true);

  // SORT RAND() if appropriate
  registerRule("remove-sort-rand", removeSortRandRule, removeSortRandRule_pass5,
               true);

  // remove unused INTO variable from COLLECT, or unused aggregates
  registerRule("remove-collect-variables", removeCollectVariablesRule,
               removeCollectVariablesRule_pass5, true);

  // remove unused out variables for data-modification queries
  registerRule("remove-data-modification-out-variables",
               removeDataModificationOutVariablesRule,
               removeDataModificationOutVariablesRule_pass5, true);

  // propagate constant attributes in FILTERs
  registerRule("propagate-constant-attributes", propagateConstantAttributesRule,
               propagateConstantAttributesRule_pass5, true);

  //////////////////////////////////////////////////////////////////////////////
  /// "Pass 6": use indexes if possible for FILTER and/or SORT nodes
  //////////////////////////////////////////////////////////////////////////////

  // try to replace simple OR conditions with IN
  registerRule("replace-or-with-in", replaceOrWithInRule,
               replaceOrWithInRule_pass6, true);

  // try to remove redundant OR conditions
  registerRule("remove-redundant-or", removeRedundantOrRule,
               removeRedundantOrRule_pass6, true);

  // try to find a filter after an enumerate collection and find indexes
  registerRule("use-indexes", useIndexesRule, useIndexesRule_pass6, true);

  // try to remove filters which are covered by index ranges
  registerRule("remove-filter-covered-by-index",
               removeFiltersCoveredByIndexRule,
               removeFiltersCoveredByIndexRule_pass6, true);

  // try to find sort blocks which are superseeded by indexes
  registerRule("use-index-for-sort", useIndexForSortRule,
               useIndexForSortRule_pass6, true);

  // sort in-values in filters (note: must come after
  // remove-filter-covered-by-index rule)
  registerRule("sort-in-values", sortInValuesRule, sortInValuesRule_pass6,
               true);

  // remove calculations that are never necessary
  registerRule("remove-unnecessary-calculations-2",
               removeUnnecessaryCalculationsRule,
               removeUnnecessaryCalculationsRule_pass6, true);

  // finally, push calculations as far down as possible
  registerRule("move-calculations-down", moveCalculationsDownRule,
               moveCalculationsDownRule_pass9, true);

// fuse calculations
#if 0
  registerRule("fuse-calculations",
               fuseCalculationsRule,
               fuseCalculationsRule_pass9,
               true);
#endif

  // patch update statements
  registerRule("patch-update-statements", patchUpdateStatementsRule,
               patchUpdateStatementsRule_pass9, true);

  if (arangodb::ServerState::instance()->isCoordinator()) {
    // distribute operations in cluster
    registerRule("scatter-in-cluster", scatterInClusterRule,
                 scatterInClusterRule_pass10, false);

    registerRule("distribute-in-cluster", distributeInClusterRule,
                 distributeInClusterRule_pass10, false);

    // distribute operations in cluster
    registerRule("distribute-filtercalc-to-cluster",
                 distributeFilternCalcToClusterRule,
                 distributeFilternCalcToClusterRule_pass10, true);

    registerRule("distribute-sort-to-cluster", distributeSortToClusterRule,
                 distributeSortToClusterRule_pass10, true);

    registerRule("remove-unnecessary-remote-scatter",
                 removeUnnecessaryRemoteScatterRule,
                 removeUnnecessaryRemoteScatterRule_pass10, true);

    registerRule("undistribute-remove-after-enum-coll",
                 undistributeRemoveAfterEnumCollRule,
                 undistributeRemoveAfterEnumCollRule_pass10, true);
  }
}