discover views on startup

arangod/MMFiles/MMFilesEngine.cpp

@@ -268,12 +268,12 @@ void MMFilesEngine::recoveryDone(TRI_vocbase_t* vocbase) {
     std::string const& name = it.first;
     std::string const& file = it.second;
 
-    LOG_TOPIC(DEBUG, arangodb::Logger::FIXME) << "collection '" << name << "' was deleted, wiping it";
+    LOG_TOPIC(DEBUG, arangodb::Logger::FIXME) << "collection/view '" << name << "' was deleted, wiping it";
 
     int res = TRI_RemoveDirectory(file.c_str());
 
     if (res != TRI_ERROR_NO_ERROR) {
-      LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "cannot wipe deleted collection '" << name << "': " << TRI_errno_string(res);
+      LOG_TOPIC(WARN, arangodb::Logger::FIXME) << "cannot wipe deleted collection/view '" << name << "': " << TRI_errno_string(res);
     }
   }
   _deleted.clear();
@@ -497,10 +497,8 @@ int MMFilesEngine::getCollectionsAndIndexes(TRI_vocbase_t* vocbase,
 
     if (!TRI_IsWritable(directory.c_str())) {
       // the collection directory we found is not writable for the current
-      // user
+      // user. this can cause serious trouble so we will abort the server start if
-      // this can cause serious trouble so we will abort the server start if
+      // we encounter this situation
-      // we
-      // encounter this situation
       LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "database subdirectory '" << directory
                 << "' is not writable for current user";
 
@@ -543,6 +541,76 @@ int MMFilesEngine::getCollectionsAndIndexes(TRI_vocbase_t* vocbase,
 
   return TRI_ERROR_NO_ERROR;
 }
+
+int MMFilesEngine::getViews(TRI_vocbase_t* vocbase, 
+                            arangodb::velocypack::Builder& result) {
+  result.openArray();
+
+  std::string const path = databaseDirectory(vocbase->id());
+  std::vector<std::string> files = TRI_FilesDirectory(path.c_str());
+
+  for (auto const& name : files) {
+    TRI_ASSERT(!name.empty());
+
+    if (!StringUtils::isPrefix(name, "view-") ||
+        StringUtils::isSuffix(name, ".tmp")) {
+      // no match, ignore this file
+      continue;
+    }
+
+    std::string const directory = FileUtils::buildFilename(path, name);
+
+    if (!TRI_IsDirectory(directory.c_str())) {
+      LOG_TOPIC(DEBUG, arangodb::Logger::FIXME) << "ignoring non-directory '" << directory << "'";
+      continue;
+    }
+
+    if (!TRI_IsWritable(directory.c_str())) {
+      // the collection directory we found is not writable for the current
+      // user. this can cause serious trouble so we will abort the server start if
+      // we encounter this situation
+      LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "database subdirectory '" << directory
+                << "' is not writable for current user";
+
+      return TRI_ERROR_ARANGO_DATADIR_NOT_WRITABLE;
+    }
+
+    int res = TRI_ERROR_NO_ERROR;
+
+    try {
+      VPackBuilder builder = loadViewInfo(vocbase, directory);
+      VPackSlice info = builder.slice();
+
+      if (VelocyPackHelper::readBooleanValue(info, "deleted", false)) {
+        std::string name = VelocyPackHelper::getStringValue(info, "name", "");
+        _deleted.emplace_back(std::make_pair(name, directory));
+        continue;
+      }
+      // add view info
+      result.add(info);
+    } catch (arangodb::basics::Exception const& e) {
+      std::string tmpfile = FileUtils::buildFilename(directory, ".tmp");
+
+      if (TRI_ExistsFile(tmpfile.c_str())) {
+        LOG_TOPIC(TRACE, arangodb::Logger::FIXME) << "ignoring temporary directory '" << tmpfile << "'";
+        // temp file still exists. this means the view was not created
+        // fully and needs to be ignored
+        continue;  // ignore this directory
+      }
+
+      res = e.code();
+
+      LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot read view info file in directory '"
+                << directory << "': " << TRI_errno_string(res);
+
+      return res;
+    }
+  }
+
+  result.close();
+
+  return TRI_ERROR_NO_ERROR;
+}
   
 void MMFilesEngine::waitForSync(TRI_voc_tick_t tick) {
   MMFilesLogfileManager::instance()->slots()->waitForTick(tick);
@@ -1692,6 +1760,38 @@ TRI_vocbase_t* MMFilesEngine::openExistingDatabase(TRI_voc_tick_t id, std::strin
                                                    bool wasCleanShutdown, bool isUpgrade) {
   auto vocbase = std::make_unique<TRI_vocbase_t>(TRI_VOCBASE_TYPE_NORMAL, id, name);
 
+  // scan the database path for views
+  try {
+    VPackBuilder builder;
+    int res = getViews(vocbase.get(), builder);
+
+    if (res != TRI_ERROR_NO_ERROR) {
+      THROW_ARANGO_EXCEPTION(res);
+    }
+    
+    VPackSlice slice = builder.slice();
+    TRI_ASSERT(slice.isArray());
+
+    for (auto const& it : VPackArrayIterator(slice)) {
+      // we found a view that is still active
+      TRI_ASSERT(!it.get("id").isNone());
+      std::shared_ptr<LogicalView> view = std::make_shared<arangodb::LogicalView>(vocbase.get(), it);
+      StorageEngine::registerView(vocbase.get(), view);
+
+      auto physical = static_cast<MMFilesView*>(view->getPhysical());
+      TRI_ASSERT(physical != nullptr);
+
+      registerViewPath(vocbase->id(), view->id(), physical->path());
+    }
+  } catch (std::exception const& ex) {
+    LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "error while opening database: " << ex.what();
+    throw;
+  } catch (...) {
+    LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "error while opening database: unknown exception";
+    throw;
+  }
+
+
   // scan the database path for collections
   try {
     VPackBuilder builder;
@@ -1998,6 +2098,44 @@ VPackBuilder MMFilesEngine::loadCollectionInfo(TRI_vocbase_t* vocbase, std::stri
   return VPackCollection::merge(slice, indexesPatch.slice(), false);
 }
 
+VPackBuilder MMFilesEngine::loadViewInfo(TRI_vocbase_t* vocbase, std::string const& path) {
+  // find parameter file
+  std::string filename =
+      arangodb::basics::FileUtils::buildFilename(path, parametersFilename());
+
+  if (!TRI_ExistsFile(filename.c_str())) {
+    filename += ".tmp";  // try file with .tmp extension
+    if (!TRI_ExistsFile(filename.c_str())) {
+      THROW_ARANGO_EXCEPTION(TRI_ERROR_ARANGO_ILLEGAL_PARAMETER_FILE);
+    }
+  }
+
+  std::shared_ptr<VPackBuilder> content =
+      arangodb::basics::VelocyPackHelper::velocyPackFromFile(filename);
+  VPackSlice slice = content->slice();
+  if (!slice.isObject()) {
+    LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot open '" << filename
+             << "', view parameters are not readable";
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_ARANGO_ILLEGAL_PARAMETER_FILE);
+  }
+
+  if (filename.substr(filename.size() - 4, 4) == ".tmp") {
+    // we got a tmp file. Now try saving the original file
+    std::string const original(filename.substr(0, filename.size() - 4));
+    bool ok = arangodb::basics::VelocyPackHelper::velocyPackToFile(original, slice, true);
+    
+    if (!ok) {
+      LOG_TOPIC(ERR, arangodb::Logger::FIXME) << "cannot store view parameters in file '" << original << "'";
+    }
+  }
+
+  VPackBuilder patch;
+  patch.openObject();
+  patch.add("path", VPackValue(path));
+  patch.close();
+  return VPackCollection::merge(slice, patch.slice(), false);
+}
+
 /// @brief remove data of expired compaction blockers
 bool MMFilesEngine::cleanupCompactionBlockers(TRI_vocbase_t* vocbase) {
   // check if we can instantly acquire the lock

arangod/MMFiles/MMFilesEngine.h

@@ -114,6 +114,8 @@ class MMFilesEngine final : public StorageEngine {
   int getCollectionsAndIndexes(TRI_vocbase_t* vocbase, arangodb::velocypack::Builder& result,
                                bool wasCleanShutdown, bool isUpgrade) override;
   
+  int getViews(TRI_vocbase_t* vocbase, arangodb::velocypack::Builder& result) override;
+  
   // return the path for a collection
   std::string collectionPath(TRI_vocbase_t const* vocbase, TRI_voc_cid_t id) const override { 
     return collectionDirectory(vocbase->id(), id); 
@@ -313,6 +315,8 @@ public:
   /// @brief transfer markers into a collection
   int transferMarkers(LogicalCollection* collection, MMFilesCollectorCache*,
                       MMFilesOperationsType const&);
+  
+  std::string viewDirectory(TRI_voc_tick_t databaseId, TRI_voc_cid_t viewId) const;
 
  private:
   /// @brief: check the initial markers in a datafile
@@ -351,7 +355,6 @@ public:
   std::string databaseParametersFilename(TRI_voc_tick_t databaseId) const;
   std::string collectionDirectory(TRI_voc_tick_t databaseId, TRI_voc_cid_t collectionId) const;
   std::string collectionParametersFilename(TRI_voc_tick_t databaseId, TRI_voc_cid_t collectionId) const;
-  std::string viewDirectory(TRI_voc_tick_t databaseId, TRI_voc_cid_t viewId) const;
   std::string viewParametersFilename(TRI_voc_tick_t databaseId, TRI_voc_cid_t viewId) const;
   std::string indexFilename(TRI_voc_tick_t databaseId, TRI_voc_cid_t collectionId, TRI_idx_iid_t indexId) const;
   std::string indexFilename(TRI_idx_iid_t indexId) const;
@@ -390,6 +393,7 @@ public:
                           bool forceSync) const;
 
   arangodb::velocypack::Builder loadCollectionInfo(TRI_vocbase_t* vocbase, std::string const& path);
+  arangodb::velocypack::Builder loadViewInfo(TRI_vocbase_t* vocbase, std::string const& path);
  
   // start the cleanup thread for the database 
   int startCleanup(TRI_vocbase_t* vocbase);

arangod/MMFiles/MMFilesView.cpp

@@ -22,52 +22,51 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "MMFilesView.h"
-#include "Aql/PlanCache.h"
-#include "Aql/QueryCache.h"
 #include "ApplicationFeatures/ApplicationServer.h"
 #include "Basics/FileUtils.h"
-#include "Basics/PerformanceLogScope.h"
 #include "Basics/ReadLocker.h"
 #include "Basics/Result.h"
-#include "Basics/StaticStrings.h"
-#include "Basics/Timers.h"
-#include "Basics/VelocyPackHelper.h"
 #include "Basics/WriteLocker.h"
-#include "Basics/encoding.h"
-#include "Basics/process-utils.h"
-#include "Cluster/ClusterMethods.h"
-#include "Cluster/CollectionLockState.h"
 #include "Logger/Logger.h"
-#include "MMFiles/MMFilesCollectionReadLocker.h"
+#include "MMFiles/MMFilesEngine.h"
-#include "MMFiles/MMFilesCollectionWriteLocker.h"
-#include "MMFiles/MMFilesDatafile.h"
-#include "MMFiles/MMFilesDatafileHelper.h"
-#include "MMFiles/MMFilesDocumentOperation.h"
-#include "MMFiles/MMFilesDocumentPosition.h"
-#include "MMFiles/MMFilesIndexElement.h"
 #include "MMFiles/MMFilesLogfileManager.h"
-#include "MMFiles/MMFilesPrimaryIndex.h"
-#include "MMFiles/MMFilesToken.h"
-#include "MMFiles/MMFilesTransactionState.h"
-#include "RestServer/DatabaseFeature.h"
-#include "Scheduler/Scheduler.h"
-#include "Scheduler/SchedulerFeature.h"
 #include "StorageEngine/EngineSelectorFeature.h"
 #include "StorageEngine/StorageEngine.h"
-#include "Transaction/Helpers.h"
-#include "Transaction/Methods.h"
-#include "Utils/CollectionNameResolver.h"
 #include "Utils/Events.h"
-#include "Utils/OperationOptions.h"
-#include "Utils/SingleCollectionTransaction.h"
-#include "Transaction/StandaloneContext.h"
-#include "VocBase/KeyGenerator.h"
-#include "VocBase/LogicalCollection.h"
 #include "VocBase/ticks.h"
-#include "Indexes/IndexIterator.h"
 
 using namespace arangodb;
-using Helper = arangodb::basics::VelocyPackHelper;
+
+MMFilesView::MMFilesView(LogicalView* view,
+                         VPackSlice const& info)
+    : PhysicalView(view, info) {}
+
+MMFilesView::MMFilesView(LogicalView* logical,
+                         PhysicalView* physical)
+    : PhysicalView(logical, VPackSlice::emptyObjectSlice()) {}
+
+MMFilesView::~MMFilesView() {
+  if (_logicalView->deleted()) {
+    try {
+      StorageEngine* engine = EngineSelectorFeature::ENGINE;
+      std::string directory = static_cast<MMFilesEngine*>(engine)->viewDirectory(_logicalView->vocbase()->id(), _logicalView->id());
+      TRI_RemoveDirectory(directory.c_str());
+    } catch (...) {
+      // must ignore errors here as we are in the destructor
+    }
+  }
+}
+
+void MMFilesView::getPropertiesVPack(velocypack::Builder& result) const {
+  TRI_ASSERT(result.isOpenObject());
+  result.add("path", VPackValue(_path));
+  TRI_ASSERT(result.isOpenObject());
+}
+  
+/// @brief opens an existing view
+void MMFilesView::open(bool ignoreErrors) {}
+
+void MMFilesView::drop() {}
 
 arangodb::Result MMFilesView::updateProperties(VPackSlice const& slice,
                                                bool doSync) {
@@ -105,23 +104,3 @@ PhysicalView* MMFilesView::clone(LogicalView* logical, PhysicalView* physical){
   return new MMFilesView(logical, physical);
 }
 
-MMFilesView::MMFilesView(LogicalView* view,
-                         VPackSlice const& info)
-    : PhysicalView(view, info) {}
-
-MMFilesView::MMFilesView(LogicalView* logical,
-                         PhysicalView* physical)
-    : PhysicalView(logical, VPackSlice::emptyObjectSlice()) {}
-
-MMFilesView::~MMFilesView() {}
-
-void MMFilesView::getPropertiesVPack(velocypack::Builder& result) const {
-  TRI_ASSERT(result.isOpenObject());
-  result.add("path", VPackValue(_path));
-  TRI_ASSERT(result.isOpenObject());
-}
-  
-/// @brief opens an existing view
-void MMFilesView::open(bool ignoreErrors) {}
-
-void MMFilesView::drop() {}

arangod/StorageEngine/StorageEngine.h

@@ -116,6 +116,8 @@ class StorageEngine : public application_features::ApplicationFeature {
   // for each database
   virtual int getCollectionsAndIndexes(TRI_vocbase_t* vocbase, arangodb::velocypack::Builder& result,
                                        bool wasCleanShutdown, bool isUpgrade) = 0;
+  
+  virtual int getViews(TRI_vocbase_t* vocbase, arangodb::velocypack::Builder& result) = 0;
 
   // return the path for a database
   virtual std::string databasePath(TRI_vocbase_t const* vocbase) const = 0;
@@ -398,6 +400,11 @@ class StorageEngine : public application_features::ApplicationFeature {
                           arangodb::LogicalCollection* collection) {
     vocbase->registerCollection(true, collection);
   }
+  
+  void registerView(TRI_vocbase_t* vocbase,
+                    std::shared_ptr<arangodb::LogicalView> view) {
+    vocbase->registerView(true, view);
+  }
 
  private:
 

arangod/VocBase/vocbase.cpp

@@ -1312,7 +1312,7 @@ std::shared_ptr<arangodb::LogicalView> TRI_vocbase_t::createViewWorker(
   registerView(basics::ConditionalLocking::DoNotLock, view);
 
   try {
-    // cid might have been assigned
+    // id might have been assigned
     id = view->id();
 
     // Let's try to persist it.
@@ -1410,6 +1410,16 @@ int TRI_vocbase_t::dropView(std::shared_ptr<arangodb::LogicalView> view) {
   arangodb::aql::QueryCache::instance()->invalidate(this);
 
   view->setDeleted(true);
+  VPackBuilder b;
+  b.openObject();
+  view->toVelocyPack(b);
+  b.close();
+
+  bool doSync =
+     application_features::ApplicationServer::getFeature<DatabaseFeature>(
+          "Database")
+          ->forceSyncProperties();
+  view->updateProperties(b.slice(), doSync); 
 
   unregisterView(view);