Backup with view (#10386)

* Commit and store view related data on create backup.
* Fixes to upgrade on env-var.
* Fixed CMakeLists.txt.
* Changes to allow restore from 3.5 to 3.6.
* Remove unnecessary code introduced in merge.
* CHANGELOG:

CHANGELOG

@@ -1,5 +1,10 @@
 devel
 -----
+
+* Include ArangoSearch data in hotbackups.
+
+* Allow to restore 3.5 hotbackups in 3.6.
+
 * Fixed ArangoSearch index removes being discarded on commiting consolidation results with
   pending removes after some segments under consolidation were already committed 
 

CMakeLists.txt

@@ -1384,3 +1384,6 @@ message(STATUS "building for git revision: ${ARANGODB_BUILD_REPOSITORY}")
 #     message(STATUS "${_variableName}=${${_variableName}}")
 #   endforeach ()
 # endif ()
+
+add_custom_target(arangodb
+        DEPENDS arangod arangosh arangodump arangoexport arangobackup arangoimport arangorestore)

arangod/CMakeLists.txt

@@ -873,6 +873,13 @@ target_link_libraries(arango_rocksdb arango_indexes)
 target_link_libraries(arango_rocksdb arango_storage_engine_common)
 target_link_libraries(arango_rocksdb boost_boost)
 
+if (USE_ENTERPRISE)
+  # this is required for hotbackup. Views need to be flushed.
+  target_include_directories(arango_rocksdb
+    PUBLIC ${IRESEARCH_INCLUDE}
+  )
+endif()
+
 target_link_libraries(arango_storage_engine arango_cluster_engine)
 target_link_libraries(arango_storage_engine arango_cluster_methods)
 target_link_libraries(arango_storage_engine arango_mmfiles)

arangod/Cluster/ClusterMethods.cpp

@@ -3011,7 +3011,8 @@ arangodb::Result hotRestoreCoordinator(ClusterFeature& feature, VPackSlice const
     using arangodb::methods::VersionResult;
 #ifdef USE_ENTERPRISE
     // Will never be called in community
-    if (!RocksDBHotBackup::versionTestRestore(meta._version)) {
+    bool autoUpgradeNeeded;   // not actually used
+    if (!RocksDBHotBackup::versionTestRestore(meta._version, autoUpgradeNeeded)) {
       return arangodb::Result(TRI_ERROR_HOT_RESTORE_INTERNAL,
                               "Version mismatch");
     }

arangod/IResearch/IResearchLink.cpp

@@ -109,31 +109,6 @@ struct LinkTrxState final : public arangodb::TransactionState::Cookie {
   }
 };
 
-////////////////////////////////////////////////////////////////////////////////
-/// @brief compute the data path to user for iresearch data store
-///        get base path from DatabaseServerFeature (similar to MMFilesEngine)
-///        the path is hardcoded to reside under:
-///        <DatabasePath>/<IResearchLink::type()>-<link id>
-///        similar to the data path calculation for collections
-////////////////////////////////////////////////////////////////////////////////
-irs::utf8_path getPersistedPath(arangodb::DatabasePathFeature const& dbPathFeature,
-                                arangodb::iresearch::IResearchLink const& link) {
-  irs::utf8_path dataPath(dbPathFeature.directory());
-  static const std::string subPath("databases");
-  static const std::string dbPath("database-");
-
-  dataPath /= subPath;
-  dataPath /= dbPath;
-  dataPath += std::to_string(link.collection().vocbase().id());
-  dataPath /= arangodb::iresearch::DATA_SOURCE_TYPE.name();
-  dataPath += "-";
-  dataPath += std::to_string(link.collection().id());  // has to be 'id' since this can be a per-shard collection
-  dataPath += "_";
-  dataPath += std::to_string(link.id());
-
-  return dataPath;
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief inserts ArangoDB document into an IResearch data store
 ////////////////////////////////////////////////////////////////////////////////
@@ -241,6 +216,31 @@ bool readTick(irs::bytes_ref const& payload, TRI_voc_tick_t& tick) noexcept {
 namespace arangodb {
 namespace iresearch {
 
+////////////////////////////////////////////////////////////////////////////////
+/// @brief compute the data path to user for iresearch data store
+///        get base path from DatabaseServerFeature (similar to MMFilesEngine)
+///        the path is hardcoded to reside under:
+///        <DatabasePath>/<IResearchLink::type()>-<link id>
+///        similar to the data path calculation for collections
+////////////////////////////////////////////////////////////////////////////////
+irs::utf8_path getPersistedPath(arangodb::DatabasePathFeature const& dbPathFeature,
+                                arangodb::iresearch::IResearchLink const& link) {
+  irs::utf8_path dataPath(dbPathFeature.directory());
+  static const std::string subPath("databases");
+  static const std::string dbPath("database-");
+
+  dataPath /= subPath;
+  dataPath /= dbPath;
+  dataPath += std::to_string(link.collection().vocbase().id());
+  dataPath /= arangodb::iresearch::DATA_SOURCE_TYPE.name();
+  dataPath += "-";
+  dataPath += std::to_string(link.collection().id());  // has to be 'id' since this can be a per-shard collection
+  dataPath += "_";
+  dataPath += std::to_string(link.id());
+
+  return dataPath;
+}
+
 IResearchLink::IResearchLink(
     TRI_idx_iid_t iid,
     LogicalCollection& collection)

arangod/IResearch/IResearchLink.h

@@ -29,10 +29,11 @@
 #include "store/directory.hpp"
 #include "utils/utf8_path.hpp"
 
+#include "Indexes/Index.h"
 #include "IResearchLinkMeta.h"
 #include "IResearchViewMeta.h"
 #include "IResearchVPackComparer.h"
-#include "Indexes/Index.h"
+#include "RestServer/DatabasePathFeature.h"
 #include "Transaction/Status.h"
 
 namespace arangodb {
@@ -333,6 +334,9 @@ class IResearchLink {
   bool _createdInRecovery; // link was created based on recovery marker
 };  // IResearchLink
 
+irs::utf8_path getPersistedPath(arangodb::DatabasePathFeature const& dbPathFeature,
+                                arangodb::iresearch::IResearchLink const& link);
+
 }  // namespace iresearch
 }  // namespace arangodb
 

arangod/RestServer/UpgradeFeature.cpp

@@ -28,8 +28,13 @@
 #include "ApplicationFeatures/SupervisorFeature.h"
 #include "Basics/application-exit.h"
 #include "Basics/ScopeGuard.h"
+#include "Basics/StaticStrings.h"
+#include "Cluster/ClusterFeature.h"
 #include "Cluster/ClusterFeature.h"
 #include "Cluster/ServerState.h"
+#ifdef USE_ENTERPRISE
+#include "Enterprise/StorageEngine/HotBackupFeature.h"
+#endif
 #include "FeaturePhases/AqlFeaturePhase.h"
 #include "GeneralServer/AuthenticationFeature.h"
 #include "Logger/LogMacros.h"
@@ -85,10 +90,8 @@ void UpgradeFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
 extern std::function<int()> * restartAction;
 
 #ifndef _WIN32
-static std::string const UPGRADE_ENV = "ARANGODB_UPGRADE_DURING_RESTORE";
-
 static int upgradeRestart() {
-  unsetenv(UPGRADE_ENV.c_str());
+  unsetenv(StaticStrings::UpgradeEnvName.c_str());
   return 0;
 }
 #endif
@@ -103,11 +106,15 @@ void UpgradeFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
   // variable at runtime and then does a restore. After the restart (with
   // the old data) the database upgrade is run and another restart is
   // happening afterwards with the environment variable being cleared.
-  char* upgrade = getenv(UPGRADE_ENV.c_str());
+  char* upgrade = getenv(StaticStrings::UpgradeEnvName.c_str());
   if (upgrade != nullptr) {
     _upgrade = true;
     restartAction = new std::function<int()>();
     *restartAction = upgradeRestart;
+    LOG_TOPIC("fdeae", INFO, Logger::STARTUP)
+        << "Detected environment variable " << StaticStrings::UpgradeEnvName
+        << " with value " << upgrade
+        << " will perform database auto-upgrade and immediately restart.";
   }
 #endif
   if (_upgrade && !_upgradeCheck) {
@@ -150,6 +157,11 @@ void UpgradeFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
 
   DatabaseFeature& database = server().getFeature<DatabaseFeature>();
   database.enableUpgrade();
+
+#ifdef USE_ENTERPRISE
+  HotBackupFeature& hotBackupFeature = server().getFeature<HotBackupFeature>();
+  hotBackupFeature.forceDisable();
+#endif
 }
 
 void UpgradeFeature::prepare() {

lib/Basics/StaticStrings.cpp

@@ -275,3 +275,7 @@ std::string const StaticStrings::RebootId("rebootId");
 
 std::string const StaticStrings::New("new");
 std::string const StaticStrings::Old("old");
+std::string const StaticStrings::UpgradeEnvName(
+    "ARANGODB_UPGRADE_DURING_RESTORE");
+std::string const StaticStrings::BackupToDeleteName("DIRECTORY_TO_DELETE");
+std::string const StaticStrings::BackupSearchToDeleteName("DIRECTORY_TO_DELETE_SEARCH");

lib/Basics/StaticStrings.h

@@ -251,6 +251,9 @@ class StaticStrings {
   static std::string const RebootId;
   static std::string const New;
   static std::string const Old;
+  static std::string const UpgradeEnvName;
+  static std::string const BackupToDeleteName;
+  static std::string const BackupSearchToDeleteName;
 };
 }  // namespace arangodb
 

tests/RocksDBEngine/HotBackupTest.cpp

@@ -217,6 +217,7 @@ public:
      arangodb::RandomGenerator::initialize(arangodb::RandomGenerator::RandomType::MERSENNE);
    }
 
+
    _id = TRI_GetTempPath();
    _id += TRI_DIR_SEPARATOR_CHAR;
    _id += "arangotest-";
@@ -330,14 +331,16 @@ public:
     pathname += "backups";
     pathname += TRI_DIR_SEPARATOR_CHAR;
     pathname += _idRestore;
+    pathname += TRI_DIR_SEPARATOR_CHAR;
+    pathname += "engine_rocksdb";
     retVal = TRI_CreateRecursiveDirectory(pathname.c_str(), systemError,
                                           systemErrorStr);
 
     EXPECT_EQ(TRI_ERROR_NO_ERROR, retVal);
 
+    writeFile(pathname.c_str(), "../META", "{\"version\":\"" ARANGODB_VERSION "\", \"datetime\":\"xxx\", \"id\":\"xxx\"}");
     writeFile(pathname.c_str(), "MANIFEST-000003", "manifest info");
     writeFile(pathname.c_str(), "CURRENT", "MANIFEST-000003\n");
-    writeFile(pathname.c_str(), "META", "{\"version\":\"" ARANGODB_VERSION "\", \"datetime\":\"xxx\", \"id\":\"xxx\"}");
     writeFile(pathname.c_str(), "IDENTITY", "huh?");
     writeFile(pathname.c_str(), "000111.sst", "raw data 1");
     writeFile(pathname.c_str(), "000111.sha.e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.hash", "");
@@ -362,7 +365,7 @@ public:
 
 /// @brief test
 TEST(RocksDBHotBackupRestoreDirectories, test_createRestoringDirectory) {
-  std::string restoringDir, tempname;
+  std::string fullRestoringDir, restoringDir, restoringSearchDir, tempname;
   bool retBool;
 
   VPackBuilder report;
@@ -371,7 +374,7 @@ TEST(RocksDBHotBackupRestoreDirectories, test_createRestoringDirectory) {
   RocksDBHotBackupRestoreTest testee(feature, VPackSlice(), report);
   testee.createHotDirectory();
 
-  retBool = testee.createRestoringDirectory(restoringDir);
+  retBool = testee.createRestoringDirectories(fullRestoringDir, restoringDir, restoringSearchDir);
 
   // spot check files in restoring dir
   EXPECT_TRUE( retBool );
@@ -391,7 +394,8 @@ TEST(RocksDBHotBackupRestoreDirectories, test_createRestoringDirectory) {
   EXPECT_TRUE( TRI_IsRegularFile(tempname.c_str()) ); // looks same as hard link
 
   // verify still present in originating dir
-  restoringDir = testee.rebuildPath(testee.getDirectoryRestore());
+  restoringDir = testee.rebuildPath(testee.getDirectoryRestore() +
+                                    TRI_DIR_SEPARATOR_CHAR + "engine_rocksdb");
   EXPECT_TRUE( TRI_ExistsFile(restoringDir.c_str()) );
   EXPECT_TRUE( TRI_IsDirectory(restoringDir.c_str()) );
   tempname = restoringDir + TRI_DIR_SEPARATOR_CHAR + "MANIFEST-000003";