fix cluster index selectivity estimates (#6470)

arangod/Cluster/ClusterInfo.cpp

@@ -762,7 +762,7 @@ void ClusterInfo::loadPlan() {
                 LOG_TOPIC(TRACE, Logger::CLUSTER) << "copy index estimates";
                 newCollection->clusterIndexEstimates(std::move(selectivity));
                 newCollection->clusterIndexEstimatesTTL(selectivityTTL);
-                for(std::shared_ptr<Index>& idx : newCollection->getIndexes()){
+                for (std::shared_ptr<Index>& idx : newCollection->getIndexes()) {
                   auto it = selectivity.find(std::to_string(idx->id()));
                   if (it != selectivity.end()) {
                     idx->updateClusterSelectivityEstimate(it->second);

arangod/Cluster/ClusterMethods.cpp

@@ -1028,10 +1028,10 @@ int selectivityEstimatesOnCoordinator(
 
         if (answer.isObject()) {
           // add to the total
-          for(auto const& identifier : VPackObjectIterator(answer.get("identifiers"))){
-            if(identifier.value.hasKey("selectivityEstimate")) {
+          for (auto const& identifier : VPackObjectIterator(answer.get("identifiers"))) {
+            if (identifier.value.hasKey("selectivityEstimate")) {
               StringRef shard_index_id(identifier.key);
-              auto split_point = std::find(shard_index_id.begin(), shard_index_id.end(),'/');
+              auto split_point = std::find(shard_index_id.begin(), shard_index_id.end(), '/');
               std::string index(split_point + 1, shard_index_id.end());
 
               double estimate = arangodb::basics::VelocyPackHelper::getNumericValue(

arangod/Graph/BaseOptions.cpp

@@ -151,9 +151,12 @@ double BaseOptions::LookupInfo::estimateCost(size_t& nrItems) const {
   TRI_ASSERT(!idxHandles.empty());
   std::shared_ptr<Index> idx = idxHandles[0].getIndex();
   if (idx->hasSelectivityEstimate()) {
-    double expected = 1 / idx->selectivityEstimate();
-    nrItems += static_cast<size_t>(expected);
-    return expected;
+    double s = idx->selectivityEstimate();
+    if (s > 0.0) {
+      double expected = 1 / s;
+      nrItems += static_cast<size_t>(expected);
+      return expected;
+    } 
   }
   // Some hard-coded value
   nrItems += 1000;

arangod/Indexes/Index.h

@@ -255,7 +255,7 @@ class Index {
       arangodb::StringRef const* extra = nullptr) const;
   
   /// @brief update the cluster selectivity estimate
-  virtual void updateClusterSelectivityEstimate(double estimate = 0.1) {
+  virtual void updateClusterSelectivityEstimate(double /*estimate*/) {
     TRI_ASSERT(false); // should never be called except on Coordinator
   }
 

arangod/Transaction/Methods.cpp

@@ -3372,7 +3372,7 @@ transaction::Methods::indexesForCollectionCoordinator(
   auto selectivity = collection->clusterIndexEstimates();
 
   // push updated values into indexes
-  for(std::shared_ptr<Index>& idx : indexes){
+  for(std::shared_ptr<Index>& idx : indexes) {
     auto it = selectivity.find(std::to_string(idx->id()));
     if (it != selectivity.end()) {
       idx->updateClusterSelectivityEstimate(it->second);

arangod/VocBase/LogicalCollection.cpp

@@ -459,7 +459,7 @@ std::unordered_map<std::string, double> LogicalCollection::clusterIndexEstimates
 
   double ctime = TRI_microtime(); // in seconds
   auto needEstimateUpdate = [this, ctime]() {
-    if(_clusterEstimates.empty()) {
+    if (_clusterEstimates.empty()) {
       LOG_TOPIC(TRACE, Logger::CLUSTER) << "update because estimate is not available";
       return true;
     } else if (ctime - _clusterEstimateTTL > 60.0) {

arangod/VocBase/Methods/Indexes.cpp

@@ -114,6 +114,9 @@ arangodb::Result Indexes::getAll(LogicalCollection const* collection,
       return Result(rv, "could not retrieve estimates");
     }
 
+    // we will merge in the index estimates later
+    flags &= ~ Index::makeFlags(Index::Serialize::Estimates);
+
     VPackBuilder tmpInner;
     auto c = ClusterInfo::instance()->getCollection(databaseName, cid);
 #ifdef USE_IRESEARCH
@@ -129,7 +132,7 @@ arangodb::Result Indexes::getAll(LogicalCollection const* collection,
       auto id = StringRef(s.get("id"));
       auto found = std::find_if(estimates.begin(),
                                 estimates.end(),
-                                [&id](std::pair<std::string,double> const& v){
+                                [&id](std::pair<std::string,double> const& v) {
                                   return id == v.first;
                                 }
                                );
@@ -451,6 +454,10 @@ Result Indexes::ensureIndex(LogicalCollection* collection,
       return Result(create ? TRI_ERROR_OUT_OF_MEMORY
                            : TRI_ERROR_ARANGO_INDEX_NOT_FOUND);
     }
+  
+    // flush estimates
+    collection->clusterIndexEstimatesTTL(0.0);
+
     // the cluster won't set a proper id value
     std::string iid = tmp.slice().get("id").copyString();
     VPackBuilder b;
@@ -573,7 +580,7 @@ Result Indexes::extractHandle(arangodb::LogicalCollection const* collection,
   return Result();
 }
 
-arangodb::Result Indexes::drop(LogicalCollection const* collection,
+arangodb::Result Indexes::drop(LogicalCollection* collection,
                                VPackSlice const& indexArg) {
   if (ExecContext::CURRENT != nullptr) {
     if (ExecContext::CURRENT->databaseAuthLevel() != auth::Level::RW ||
@@ -590,6 +597,9 @@ arangodb::Result Indexes::drop(LogicalCollection const* collection,
     if (!res.ok()) {
       return res;
     }
+    
+    // flush estimates
+    collection->clusterIndexEstimatesTTL(0.0);
 
 #ifdef USE_ENTERPRISE
     return Indexes::dropCoordinatorEE(collection, iid);

arangod/VocBase/Methods/Indexes.h

@@ -60,7 +60,7 @@ struct Indexes {
       velocypack::Slice const& definition, bool create,
       velocypack::Builder& output);
 
-  static arangodb::Result drop(LogicalCollection const* collection,
+  static arangodb::Result drop(LogicalCollection* collection,
                                velocypack::Slice const& indexArg);
 
   static arangodb::Result extractHandle(