curvebs/mds: add poolset rules to support assigning poolset based on directories

Signed-off-by: NHanqing Wu <wuhanqing@corp.netease.com>

conf/mds.conf

@@ -202,3 +202,15 @@ mds.common.logDir=./  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE
 # 单元测试情况下
 # mds.common.logDir=./runlog/
 
+#
+## poolset rules
+#
+# for backward compatibility, rules are applied for select poolset when creating file
+#
+# for example
+#   mds.poolset.rules=/dir1/:poolset1;/dir2/:poolset2;/dir1/sub/:sub
+#
+# when creating file reqeust doesn't have poolset, above rules are used to select poolset
+# - if filename is /dir1/file, then poolset1 is select
+# - if filename is /dir1/sub/file, then sub is select
+mds.poolset.rules=

src/mds/nameserver2/BUILD

@@ -15,26 +15,26 @@
 #
 
 cc_library(
-    name="nameserver2",
-    hdrs=glob(["*.h"]),
-    srcs=glob(["*.cpp"]),
+    name = "nameserver2",
+    srcs = glob(["*.cpp"]),
+    hdrs = glob(["*.h"]),
     visibility = ["//visibility:public"],
     deps = [
-            "//external:glog",
-            "//external:gflags",
-            "//external:leveldb",
-            "//external:brpc",
-            "//proto:nameserver2_cc_proto",
-            "//src/common:curve_common",
-            "//src/mds/topology:topology",
-            "//src/mds/nameserver2/allocstatistic:alloc_statistic",
-            "//src/mds/chunkserverclient:chunkserverclient",
-            "//src/mds/snapshotcloneclient:snapshotcloneclient",
-            "//src/mds/common:mds_common",
-            "//src/mds/nameserver2/helper:helper",
-            "//src/mds/nameserver2/idgenerator:idgenerator",
-            "//src/common:curve_auth",
-            "//src/kvstorageclient:kvstorage_client",
+        "//external:brpc",
+        "//external:gflags",
+        "//external:glog",
+        "//external:leveldb",
+        "//proto:nameserver2_cc_proto",
+        "//src/common:curve_auth",
+        "//src/common:curve_common",
+        "//src/kvstorageclient:kvstorage_client",
+        "//src/mds/chunkserverclient",
+        "//src/mds/common:mds_common",
+        "//src/mds/nameserver2/allocstatistic:alloc_statistic",
+        "//src/mds/nameserver2/helper",
+        "//src/mds/nameserver2/idgenerator",
+        "//src/mds/snapshotcloneclient",
+        "//src/mds/topology",
+        "@com_google_absl//absl/strings",
     ],
 )
-

src/mds/nameserver2/curvefs.cpp

@@ -34,6 +34,8 @@
 #include "src/mds/common/mds_define.h"
 #include "src/mds/nameserver2/helper/namespace_helper.h"
 
+#include "absl/strings/match.h"
+
 using curve::common::TimeUtility;
 using curve::common::kDefaultPoolsetName;
 using curve::mds::topology::LogicalPool;
@@ -122,6 +124,7 @@ bool CurveFS::Init(std::shared_ptr<NameServerStorage> storage,
     maxFileLength_ = curveFSOptions.maxFileLength;
     topology_ = topology;
     snapshotCloneClient_ = snapshotCloneClient;
+    poolsetRules_ = curveFSOptions.poolsetRules;
 
     InitRootFile();
     bool ret = InitRecycleBinDir();
@@ -245,7 +248,7 @@ StatusCode CurveFS::CreateFile(const std::string& fileName,
 
     // check param
     if (filetype == FileType::INODE_PAGEFILE) {
-        StatusCode retCode = CheckOrAssignPoolset(&poolset);
+        StatusCode retCode = CheckOrAssignPoolset(fileName, &poolset);
         if (retCode != StatusCode::kOK) {
             return retCode;
         }
@@ -738,7 +741,8 @@ StatusCode CurveFS::DeleteFile(const std::string & filename, uint64_t fileId,
     }
 }
 
-StatusCode CurveFS::CheckOrAssignPoolset(std::string* poolset) const {
+StatusCode CurveFS::CheckOrAssignPoolset(const std::string& filename,
+                                         std::string* poolset) const {
     const auto names = topology_->GetPoolsetNameInCluster();
     if (names.empty()) {
         LOG(WARNING) << "Cluster doesn't have poolsets";
@@ -746,9 +750,8 @@ StatusCode CurveFS::CheckOrAssignPoolset(std::string* poolset) const {
     }
 
     if (poolset->empty()) {
-        *poolset = kDefaultPoolsetName;
-        LOG(INFO) << "Poolset is empty, set to: " << kDefaultPoolsetName;
-        return StatusCode::kOK;
+        *poolset = SelectPoolsetByRules(filename, poolsetRules_);
+        LOG(INFO) << "Poolset is empty, set to: " << *poolset;
     }
 
     auto it = std::find(names.begin(), names.end(), *poolset);
@@ -1725,7 +1728,7 @@ StatusCode CurveFS::CreateCloneFile(const std::string &fileName,
         return ret;
     }
 
-    ret = CheckOrAssignPoolset(&poolset);
+    ret = CheckOrAssignPoolset(fileName, &poolset);
     if (ret != StatusCode::kOK) {
         return ret;
     }
@@ -2479,5 +2482,29 @@ uint64_t GetOpenFileNum(void *varg) {
 bvar::PassiveStatus<uint64_t> g_open_file_num_bvar(
                         CURVE_MDS_CURVEFS_METRIC_PREFIX, "open_file_num",
                         GetOpenFileNum, &kCurveFS);
+
+std::string SelectPoolsetByRules(
+        const std::string& filename,
+        const std::map<std::string, std::string>& rules) {
+    if (rules.empty()) {
+        return kDefaultPoolsetName;
+    }
+
+    // using reverse order, so that we support subdir rules
+    //
+    // for example
+    //   /A/    -> poolset1
+    //   /A/B/  -> poolset2
+    //
+    // if filename is /A/B/C, then we select `poolset2`
+    for (auto it = rules.rbegin(); it != rules.rend(); ++it) {
+        if (absl::StartsWith(filename, it->first)) {
+            return it->second;
+        }
+    }
+
+    return kDefaultPoolsetName;
+}
+
 }   // namespace mds
 }   // namespace curve

src/mds/nameserver2/curvefs.h

@@ -30,6 +30,7 @@
 #include <thread>  //NOLINT
 #include <chrono>  //NOLINT
 #include <unordered_map>
+#include <map>
 #include "proto/nameserver2.pb.h"
 #include "src/mds/nameserver2/namespace_storage.h"
 #include "src/mds/common/mds_define.h"
@@ -62,6 +63,7 @@ struct CurveFSOption {
     uint64_t maxFileLength;
     RootAuthOption authOptions;
     FileRecordOptions fileRecordOptions;
+    std::map<std::string, std::string> poolsetRules;
 };
 
 struct AllocatedSize {
@@ -222,7 +224,8 @@ class CurveFS {
      *  @param[in|out] poolset: poolset name
      *  @return StatusCode::kOK if success
      */
-    StatusCode CheckOrAssignPoolset(std::string* poolset) const;
+    StatusCode CheckOrAssignPoolset(const std::string& filename,
+                                    std::string* poolset) const;
 
     /**
      *  @brief get information of all files in the directory
@@ -791,6 +794,8 @@ class CurveFS {
     uint64_t minFileLength_;
     uint64_t maxFileLength_;
     std::chrono::steady_clock::time_point startTime_;
+
+    std::map<std::string, std::string> poolsetRules_;
 };
 extern CurveFS &kCurveFS;
 
@@ -813,6 +818,10 @@ StatusCode CheckStripeParam(uint64_t segmentSize,
                             uint64_t stripeUnit,
                             uint64_t stripeCount);
 
+std::string SelectPoolsetByRules(
+        const std::string& filename,
+        const std::map<std::string, std::string>& rules);
+
 }   // namespace mds
 }   // namespace curve
 #endif   // SRC_MDS_NAMESERVER2_CURVEFS_H_

src/mds/server/mds.cpp

@@ -21,10 +21,13 @@
  */
 
 #include <glog/logging.h>
+#include <map>
 #include "src/mds/server/mds.h"
 #include "src/mds/nameserver2/helper/namespace_helper.h"
 #include "src/mds/topology/topology_storge_etcd.h"
 
+#include "absl/strings/str_split.h"
+
 using ::curve::mds::topology::TopologyStorageEtcd;
 using ::curve::mds::topology::TopologyStorageCodec;
 using ::curve::mds::topology::ChunkServerRegistInfoBuilder;
@@ -470,8 +473,11 @@ void MDS::InitCurveFSOptions(CurveFSOption *curveFSOptions) {
     FileRecordOptions fileRecordOptions;
     InitFileRecordOptions(&curveFSOptions->fileRecordOptions);
 
-    RootAuthOption authOptions;
     InitAuthOptions(&curveFSOptions->authOptions);
+
+    LOG_IF(FATAL, !ParsePoolsetRules(conf_->GetStringValue("mds.poolset.rules"),
+                                     &curveFSOptions->poolsetRules))
+            << "Fail to parse poolset rules";
 }
 
 void MDS::InitCleanManager() {
@@ -580,5 +586,32 @@ void MDS::InitHeartbeatOption(HeartbeatOption* heartbeatOption) {
     conf_->GetValueFatalIfFail("mds.heartbeat.clean_follower_afterMs",
                         &heartbeatOption->cleanFollowerAfterMs);
 }
+
+bool ParsePoolsetRules(const std::string& str,
+                       std::map<std::string, std::string>* rules) {
+    rules->clear();
+
+    if (str.empty()) {
+        return true;
+    }
+
+    for (absl::string_view sp : absl::StrSplit(str, ';')) {
+        rules->insert(absl::StrSplit(sp, ':'));
+    }
+
+    for (const auto& rule : *rules) {
+        const auto& key = rule.first;
+
+        if (key.empty() || key.front() != '/' || key.back() != '/') {
+            LOG(ERROR) << "Invalid poolset rules, key must starts and ends "
+                          "with `/`, rules: `"
+                       << str << "`";
+            return false;
+        }
+    }
+
+    return true;
+}
+
 }  // namespace mds
 }  // namespace curve

src/mds/server/mds.h

@@ -29,6 +29,7 @@
 #include <brpc/server.h>
 #include <string>
 #include <memory>
+#include <map>
 
 #include "src/mds/nameserver2/namespace_storage.h"
 #include "src/mds/nameserver2/namespace_service.h"
@@ -230,6 +231,9 @@ class MDS {
     std::shared_ptr<SnapshotCloneClient> snapshotCloneClient_;
 };
 
+bool ParsePoolsetRules(const std::string& str,
+                       std::map<std::string, std::string>* rules);
+
 }  // namespace mds
 }  // namespace curve
 

test/mds/nameserver2/curvefs_test.cpp

@@ -350,6 +350,45 @@ TEST_F(CurveFSTest, testCreateFileWithPoolset) {
     }
 }
 
+TEST(TestSelectPoolsetByRules, Test) {
+    ASSERT_EQ(kDefaultPoolsetName, SelectPoolsetByRules("/filename", {}));
+
+    {
+        std::map<std::string, std::string> rules{
+            {"/system/", "system"}
+        };
+        ASSERT_EQ("system", SelectPoolsetByRules("/system/file", rules));
+    }
+
+    {
+        std::map<std::string, std::string> rules{
+                {"/system/", "system"}
+        };
+        ASSERT_EQ(kDefaultPoolsetName, SelectPoolsetByRules("/systems", rules));
+    }
+
+    {
+        std::map<std::string, std::string> rules{
+                {"/system/", "system"},
+                {"/systems/", "system1"},
+        };
+        ASSERT_EQ("system1", SelectPoolsetByRules("/systems/file", rules));
+    }
+
+    // subdir rules
+    {
+        std::map<std::string, std::string> rules{
+                {"/system/", "system"},
+                {"/system/sub/", "system-sub"}
+        };
+        ASSERT_EQ("system-sub",
+                  SelectPoolsetByRules("/system/sub/file", rules));
+
+        ASSERT_EQ("system-sub",
+                  SelectPoolsetByRules("/system/sub/sub/file", rules));
+    }
+}
+
 TEST_F(CurveFSTest, testGetFileInfo) {
     // test parm error
     FileInfo fileInfo;

test/mds/server/mds_test.cpp

@@ -217,5 +217,42 @@ TEST_F(MDSTest, common) {
     ASSERT_LE(stopTime - startTime, 100);
 }
 
+TEST(TestParsePoolsetRules, Test) {
+    std::map<std::string, std::string> rules;
+
+    {
+        ASSERT_TRUE(ParsePoolsetRules("", &rules));
+        ASSERT_TRUE(rules.empty());
+    }
+
+    {
+        ASSERT_TRUE(ParsePoolsetRules("/:hello", &rules));
+        ASSERT_EQ(1, rules.size());
+        ASSERT_EQ("hello", rules["/"]);
+    }
+
+    {
+        ASSERT_TRUE(ParsePoolsetRules("/system/:system;/data/:data", &rules));
+        ASSERT_EQ(2, rules.size());
+        ASSERT_EQ("system", rules["/system/"]);
+        ASSERT_EQ("data", rules["/data/"]);
+    }
+
+    {
+        // key must starts and ends with '/'
+        ASSERT_FALSE(ParsePoolsetRules("/system:system;/data/:data", &rules));
+    }
+
+    {
+        // subdir rules
+        ASSERT_TRUE(ParsePoolsetRules(
+                "/system/:system;/data/:data;/system/sub/:system-sub", &rules));
+        ASSERT_EQ(3, rules.size());
+        ASSERT_EQ("system", rules["/system/"]);
+        ASSERT_EQ("data", rules["/data/"]);
+        ASSERT_EQ("system-sub", rules["/system/sub/"]);
+    }
+}
+
 }  // namespace mds
 }  // namespace curve