Skip to content

M
milvus

milvus / milvus
11 个月 前同步成功

通知 261
Star 22476
Fork 2472
M
milvus

体验新版 GitCode，发现更多精彩内容 >>

未验证 提交 295105ff 编写于 作者: X XuPeng-SH 提交者： GitHub
浏览文件
操作

Snapshot code update (#2497) 

* (db/snapshot): Add LSN in Resource
Signed-off-by: Npeng.xu <peng.xu@zilliz.com>

* (db/snapshot): add AddStepWithLsn
Signed-off-by: Npeng.xu <peng.xu@zilliz.com>

* (db/snapshot): Add Lsn related logic
Signed-off-by: Npeng.xu <peng.xu@zilliz.com>

* (db/snapshot): add operator << for Operation
Signed-off-by: Npeng.xu <peng.xu@zilliz.com>

* (db/snapshot): fix lint errors
Signed-off-by: Npeng.xu <peng.xu@zilliz.com>

* (db/snapshot): add more test
Signed-off-by: Npeng.xu <peng.xu@zilliz.com>

* (db/snapshot): operation rollback api placeholder
Signed-off-by: Npeng.xu <peng.xu@zilliz.com>

* (db/snapshot): remove hard code status code
Signed-off-by: Npeng.xu <peng.xu@zilliz.com>

* (db/snapshot): support more printable dump for some operations
Signed-off-by: Npeng.xu <peng.xu@zilliz.com>

* (db/snapshot): add ToString for Context
Signed-off-by: Npeng.xu <peng.xu@zilliz.com>

* (db/snapshot): more friendly context print
Signed-off-by: Npeng.xu <peng.xu@zilliz.com>

* (db/snapshot): rename some apis
Signed-off-by: Npeng.xu <peng.xu@zilliz.com>
上级 530ee76a
隐藏空白更改
内联 并排
Showing with 571 addition and 213 deletion
core/src/db/snapshot/CompoundOperations.cpp
浏览文件 @ 295105ff
...@@ -19,35 +19,40 @@ namespace milvus { ...@@ -19,35 +19,40 @@ namespace milvus {
namespace engine { namespace engine {
namespace snapshot { namespace snapshot {
BuildOperation::BuildOperation(const OperationContext& context, ScopedSnapshotT prev_ss) : BaseT(context, prev_ss) { CompoundBaseOperation::CompoundBaseOperation(const OperationContext& context, ScopedSnapshotT prev_ss)
: BaseT(context, prev_ss, OperationsType::W_Compound) {
} }
BuildOperation::BuildOperation(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id) CompoundBaseOperation::CompoundBaseOperation(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id)
: BaseT(context, collection_id, commit_id) { : BaseT(context, collection_id, commit_id, OperationsType::W_Compound) {
} }
std::string std::string
BuildOperation::OperationRepr() const { CompoundBaseOperation::GetRepr() const {
std::stringstream ss; std::stringstream ss;
ss << "<BO(SS=" << prev_ss_->GetID() << ",SEG="; ss << "<" << GetName() << "(";
if (context_.new_segment_files.size() == 0) { if (prev_ss_) {
ss << "?"; ss << "SS=" << prev_ss_ << GetID();
} else {
ss << context_.new_segment_files[0]->GetSegmentId();
ss << ",NSF=[";
bool first = true;
for (auto& f : context_.new_segment_files) {
if (!first) {
ss << ",";
}
ss << f->GetID();
first = false;
}
ss << "]";
} }
ss << "," << context_.ToString();
ss << ",LSN=" << GetContextLsn();
ss << ")>"; ss << ")>";
return ss.str(); return ss.str();
} }
Status
CompoundBaseOperation::PreCheck() {
if (GetContextLsn() <= prev_ss_->GetMaxLsn()) {
return Status(SS_INVALID_CONTEX_ERROR, "Invalid LSN found in operation");
}
return Status::OK();
}
BuildOperation::BuildOperation(const OperationContext& context, ScopedSnapshotT prev_ss) : BaseT(context, prev_ss) {
}
BuildOperation::BuildOperation(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id)
: BaseT(context, collection_id, commit_id) {
}
Status Status
BuildOperation::DoExecute(Store& store) { BuildOperation::DoExecute(Store& store) {
auto status = CheckStale(std::bind(&BuildOperation::CheckSegmentStale, this, std::placeholders::_1, auto status = CheckStale(std::bind(&BuildOperation::CheckSegmentStale, this, std::placeholders::_1,
...@@ -60,30 +65,30 @@ BuildOperation::DoExecute(Store& store) { ...@@ -60,30 +65,30 @@ BuildOperation::DoExecute(Store& store) {
status = op.GetResource(context_.new_segment_commit); status = op.GetResource(context_.new_segment_commit);
if (!status.ok()) if (!status.ok())
return status; return status;
AddStep(*context_.new_segment_commit); AddStepWithLsn(*context_.new_segment_commit, context_.lsn);
PartitionCommitOperation pc_op(context_, prev_ss_); PartitionCommitOperation pc_op(context_, prev_ss_);
pc_op(store); pc_op(store);
OperationContext cc_context; OperationContext cc_context;
status = pc_op.GetResource(cc_context.new_partition_commit); status = pc_op.GetResource(cc_context.new_partition_commit);
if (!status.ok()) if (!status.ok())
return status; return status;
AddStep(*cc_context.new_partition_commit); AddStepWithLsn(*cc_context.new_partition_commit, context_.lsn);
context_.new_partition_commit = cc_context.new_partition_commit;
PartitionCommitPtr pc; status = pc_op.GetResource(context_.new_partition_commit);
status = pc_op.GetResource(pc);
if (!status.ok()) if (!status.ok())
return status; return status;
AddStep(*pc); AddStepWithLsn(*context_.new_partition_commit, context_.lsn);
CollectionCommitOperation cc_op(cc_context, prev_ss_); CollectionCommitOperation cc_op(cc_context, prev_ss_);
cc_op(store); cc_op(store);
CollectionCommitPtr cc; status = cc_op.GetResource(context_.new_collection_commit);
status = cc_op.GetResource(cc);
if (!status.ok()) if (!status.ok())
return status; return status;
AddStep(*cc); AddStepWithLsn(*context_.new_collection_commit, context_.lsn);
return status; return status;
} }
...@@ -111,7 +116,7 @@ BuildOperation::CommitNewSegmentFile(const SegmentFileContext& context, SegmentF ...@@ -111,7 +116,7 @@ BuildOperation::CommitNewSegmentFile(const SegmentFileContext& context, SegmentF
if (!status.ok()) if (!status.ok())
return status; return status;
context_.new_segment_files.push_back(created); context_.new_segment_files.push_back(created);
AddStep(*created); AddStepWithLsn(*created, context_.lsn);
return status; return status;
} }
...@@ -137,7 +142,7 @@ NewSegmentOperation::DoExecute(Store& store) { ...@@ -137,7 +142,7 @@ NewSegmentOperation::DoExecute(Store& store) {
status = op.GetResource(context_.new_segment_commit); status = op.GetResource(context_.new_segment_commit);
if (!status.ok()) if (!status.ok())
return status; return status;
AddStep(*context_.new_segment_commit); AddStepWithLsn(*context_.new_segment_commit, context_.lsn);
OperationContext cc_context; OperationContext cc_context;
...@@ -148,17 +153,17 @@ NewSegmentOperation::DoExecute(Store& store) { ...@@ -148,17 +153,17 @@ NewSegmentOperation::DoExecute(Store& store) {
status = pc_op.GetResource(cc_context.new_partition_commit); status = pc_op.GetResource(cc_context.new_partition_commit);
if (!status.ok()) if (!status.ok())
return status; return status;
AddStep(*cc_context.new_partition_commit); AddStepWithLsn(*cc_context.new_partition_commit, context_.lsn);
context_.new_partition_commit = cc_context.new_partition_commit;
CollectionCommitOperation cc_op(cc_context, prev_ss_); CollectionCommitOperation cc_op(cc_context, prev_ss_);
status = cc_op(store); status = cc_op(store);
if (!status.ok()) if (!status.ok())
return status; return status;
CollectionCommitPtr cc; status = cc_op.GetResource(context_.new_collection_commit);
status = cc_op.GetResource(cc);
if (!status.ok()) if (!status.ok())
return status; return status;
AddStep(*cc); AddStepWithLsn(*context_.new_collection_commit, context_.lsn);
return status; return status;
} }
...@@ -173,7 +178,7 @@ NewSegmentOperation::CommitNewSegment(SegmentPtr& created) { ...@@ -173,7 +178,7 @@ NewSegmentOperation::CommitNewSegment(SegmentPtr& created) {
if (!status.ok()) if (!status.ok())
return status; return status;
created = context_.new_segment; created = context_.new_segment;
AddStep(*created); AddStepWithLsn(*created, context_.lsn);
return status; return status;
} }
...@@ -189,7 +194,7 @@ NewSegmentOperation::CommitNewSegmentFile(const SegmentFileContext& context, Seg ...@@ -189,7 +194,7 @@ NewSegmentOperation::CommitNewSegmentFile(const SegmentFileContext& context, Seg
status = new_sf_op->GetResource(created); status = new_sf_op->GetResource(created);
if (!status.ok()) if (!status.ok())
return status; return status;
AddStep(*created); AddStepWithLsn(*created, context_.lsn);
context_.new_segment_files.push_back(created); context_.new_segment_files.push_back(created);
return status; return status;
} }
...@@ -200,44 +205,6 @@ MergeOperation::MergeOperation(const OperationContext& context, ID_TYPE collecti ...@@ -200,44 +205,6 @@ MergeOperation::MergeOperation(const OperationContext& context, ID_TYPE collecti
: BaseT(context, collection_id, commit_id) { : BaseT(context, collection_id, commit_id) {
} }
std::string
MergeOperation::OperationRepr() const {
std::stringstream ss;
ss << "<MO(SS=" << prev_ss_->GetID() << ",SSEG=[";
{
bool first = true;
for (auto& r : context_.stale_segments) {
if (!first) {
ss << ",";
}
ss << r->GetID();
first = false;
}
}
ss << "]";
ss << ",NSEG=";
if (context_.new_segment) {
ss << context_.new_segment->GetID();
} else {
ss << "?";
}
ss << ",NSF=[";
{
bool first = true;
for (auto& f : context_.new_segment_files) {
if (!first) {
ss << ",";
}
ss << f->GetID();
first = false;
}
}
ss << "]";
ss << ")>";
return ss.str();
}
Status Status
MergeOperation::CommitNewSegment(SegmentPtr& created) { MergeOperation::CommitNewSegment(SegmentPtr& created) {
Status status; Status status;
...@@ -253,7 +220,7 @@ MergeOperation::CommitNewSegment(SegmentPtr& created) { ...@@ -253,7 +220,7 @@ MergeOperation::CommitNewSegment(SegmentPtr& created) {
if (!status.ok()) if (!status.ok())
return status; return status;
created = context_.new_segment; created = context_.new_segment;
AddStep(*created); AddStepWithLsn(*created, context_.lsn);
return status; return status;
} }
...@@ -275,7 +242,7 @@ MergeOperation::CommitNewSegmentFile(const SegmentFileContext& context, SegmentF ...@@ -275,7 +242,7 @@ MergeOperation::CommitNewSegmentFile(const SegmentFileContext& context, SegmentF
if (!status.ok()) if (!status.ok())
return status; return status;
context_.new_segment_files.push_back(created); context_.new_segment_files.push_back(created);
AddStep(*created); AddStepWithLsn(*created, context_.lsn);
return status; return status;
} }
...@@ -292,7 +259,7 @@ MergeOperation::DoExecute(Store& store) { ...@@ -292,7 +259,7 @@ MergeOperation::DoExecute(Store& store) {
status = op.GetResource(context_.new_segment_commit); status = op.GetResource(context_.new_segment_commit);
if (!status.ok()) if (!status.ok())
return status; return status;
AddStep(*context_.new_segment_commit); AddStepWithLsn(*context_.new_segment_commit, context_.lsn);
PartitionCommitOperation pc_op(context_, prev_ss_); PartitionCommitOperation pc_op(context_, prev_ss_);
status = pc_op(store); status = pc_op(store);
...@@ -303,23 +270,25 @@ MergeOperation::DoExecute(Store& store) { ...@@ -303,23 +270,25 @@ MergeOperation::DoExecute(Store& store) {
status = pc_op.GetResource(cc_context.new_partition_commit); status = pc_op.GetResource(cc_context.new_partition_commit);
if (!status.ok()) if (!status.ok())
return status; return status;
AddStep(*cc_context.new_partition_commit); AddStepWithLsn(*cc_context.new_partition_commit, context_.lsn);
context_.new_partition_commit = cc_context.new_partition_commit;
CollectionCommitOperation cc_op(cc_context, prev_ss_); CollectionCommitOperation cc_op(cc_context, prev_ss_);
status = cc_op(store); status = cc_op(store);
if (!status.ok()) if (!status.ok())
return status; return status;
CollectionCommitPtr cc; status = cc_op.GetResource(context_.new_collection_commit);
status = cc_op.GetResource(cc);
if (!status.ok()) if (!status.ok())
return status; return status;
AddStep(*cc); AddStepWithLsn(*context_.new_collection_commit, context_.lsn);
return status; return status;
} }
GetSnapshotIDsOperation::GetSnapshotIDsOperation(ID_TYPE collection_id, bool reversed) GetSnapshotIDsOperation::GetSnapshotIDsOperation(ID_TYPE collection_id, bool reversed)
: BaseT(OperationContext(), ScopedSnapshotT()), collection_id_(collection_id), reversed_(reversed) { : BaseT(OperationContext(), ScopedSnapshotT(), OperationsType::O_Compound),
collection_id_(collection_id),
reversed_(reversed) {
} }
Status Status
...@@ -349,23 +318,35 @@ GetCollectionIDsOperation::GetIDs() const { ...@@ -349,23 +318,35 @@ GetCollectionIDsOperation::GetIDs() const {
} }
DropPartitionOperation::DropPartitionOperation(const PartitionContext& context, ScopedSnapshotT prev_ss) DropPartitionOperation::DropPartitionOperation(const PartitionContext& context, ScopedSnapshotT prev_ss)
: BaseT(OperationContext(), prev_ss), context_(context) { : BaseT(OperationContext(), prev_ss), c_context_(context) {
}
std::string
DropPartitionOperation::GetRepr() const {
std::stringstream ss;
ss << "<DPO(SS=" << prev_ss_->GetID();
ss << "," << c_context_.ToString();
ss << "," << context_.ToString();
ss << ",LSN=" << GetContextLsn();
ss << ")>";
return ss.str();
} }
Status Status
DropPartitionOperation::DoExecute(Store& store) { DropPartitionOperation::DoExecute(Store& store) {
Status status; Status status;
PartitionPtr p; PartitionPtr p;
auto id = context_.id; auto id = c_context_.id;
if (id == 0) { if (id == 0) {
status = prev_ss_->GetPartitionId(context_.name, id); status = prev_ss_->GetPartitionId(c_context_.name, id);
context_.id = id; c_context_.id = id;
} }
if (!status.ok()) if (!status.ok())
return status; return status;
auto p_c = prev_ss_->GetPartitionCommitByPartitionId(id); auto p_c = prev_ss_->GetPartitionCommitByPartitionId(id);
if (!p_c) if (!p_c)
return Status(SS_NOT_FOUND_ERROR, "No partition commit found"); return Status(SS_NOT_FOUND_ERROR, "No partition commit found");
context_.stale_partition_commit = p_c;
OperationContext op_ctx; OperationContext op_ctx;
op_ctx.stale_partition_commit = p_c; op_ctx.stale_partition_commit = p_c;
...@@ -373,12 +354,11 @@ DropPartitionOperation::DoExecute(Store& store) { ...@@ -373,12 +354,11 @@ DropPartitionOperation::DoExecute(Store& store) {
status = op(store); status = op(store);
if (!status.ok()) if (!status.ok())
return status; return status;
CollectionCommitPtr cc; status = op.GetResource(context_.new_collection_commit);
status = op.GetResource(cc);
if (!status.ok()) if (!status.ok())
return status; return status;
AddStep(*cc); AddStepWithLsn(*context_.new_collection_commit, c_context_.lsn);
return status; return status;
} }
...@@ -392,7 +372,10 @@ CreatePartitionOperation::CreatePartitionOperation(const OperationContext& conte ...@@ -392,7 +372,10 @@ CreatePartitionOperation::CreatePartitionOperation(const OperationContext& conte
Status Status
CreatePartitionOperation::PreCheck() { CreatePartitionOperation::PreCheck() {
Status status; Status status = BaseT::PreCheck();
if (!status.ok()) {
return status;
}
if (!context_.new_partition) { if (!context_.new_partition) {
status = Status(SS_INVALID_CONTEX_ERROR, "No partition specified before push partition"); status = Status(SS_INVALID_CONTEX_ERROR, "No partition specified before push partition");
} }
...@@ -410,7 +393,7 @@ CreatePartitionOperation::CommitNewPartition(const PartitionContext& context, Pa ...@@ -410,7 +393,7 @@ CreatePartitionOperation::CommitNewPartition(const PartitionContext& context, Pa
if (!status.ok()) if (!status.ok())
return status; return status;
context_.new_partition = partition; context_.new_partition = partition;
AddStep(*partition); AddStepWithLsn(*partition, context_.lsn);
return status; return status;
} }
...@@ -434,10 +417,10 @@ CreatePartitionOperation::DoExecute(Store& store) { ...@@ -434,10 +417,10 @@ CreatePartitionOperation::DoExecute(Store& store) {
status = pc_op.GetResource(pc); status = pc_op.GetResource(pc);
if (!status.ok()) if (!status.ok())
return status; return status;
/* status = store.CreateResource<PartitionCommit>(PartitionCommit(collection->GetID(), partition->GetID()), pc); */ AddStepWithLsn(*pc, context_.lsn);
AddStep(*pc);
OperationContext cc_context; OperationContext cc_context;
cc_context.new_partition_commit = pc; cc_context.new_partition_commit = pc;
context_.new_partition_commit = pc;
auto cc_op = CollectionCommitOperation(cc_context, prev_ss_); auto cc_op = CollectionCommitOperation(cc_context, prev_ss_);
status = cc_op(store); status = cc_op(store);
if (!status.ok()) if (!status.ok())
...@@ -446,39 +429,58 @@ CreatePartitionOperation::DoExecute(Store& store) { ...@@ -446,39 +429,58 @@ CreatePartitionOperation::DoExecute(Store& store) {
status = cc_op.GetResource(cc); status = cc_op.GetResource(cc);
if (!status.ok()) if (!status.ok())
return status; return status;
AddStep(*cc); AddStepWithLsn(*cc, context_.lsn);
context_.new_collection_commit = cc;
return status; return status;
} }
CreateCollectionOperation::CreateCollectionOperation(const CreateCollectionContext& context) CreateCollectionOperation::CreateCollectionOperation(const CreateCollectionContext& context)
: BaseT(OperationContext(), ScopedSnapshotT()), context_(context) { : BaseT(OperationContext(), ScopedSnapshotT()), c_context_(context) {
}
Status
CreateCollectionOperation::PreCheck() {
// TODO
return Status::OK();
}
std::string
CreateCollectionOperation::GetRepr() const {
std::stringstream ss;
ss << "<CCO(";
ss << c_context_.ToString();
ss << "," << context_.ToString();
ss << ",LSN=" << GetContextLsn();
ss << ")>";
return ss.str();
} }
Status Status
CreateCollectionOperation::DoExecute(Store& store) { CreateCollectionOperation::DoExecute(Store& store) {
// TODO: Do some checks // TODO: Do some checks
CollectionPtr collection; CollectionPtr collection;
auto status = store.CreateCollection(Collection(context_.collection->GetName()), collection); auto status = store.CreateCollection(Collection(c_context_.collection->GetName()), collection);
if (!status.ok()) { if (!status.ok()) {
std::cerr << status.ToString() << std::endl; std::cerr << status.ToString() << std::endl;
return status; return status;
} }
AddStep(*collection); AddStepWithLsn(*collection, c_context_.lsn);
context_.new_collection = collection;
MappingT field_commit_ids = {}; MappingT field_commit_ids = {};
auto field_idx = 0; auto field_idx = 0;
for (auto& field_kv : context_.fields_schema) { for (auto& field_kv : c_context_.fields_schema) {
field_idx++; field_idx++;
auto& field_schema = field_kv.first; auto& field_schema = field_kv.first;
auto& field_elements = field_kv.second; auto& field_elements = field_kv.second;
FieldPtr field; FieldPtr field;
status = store.CreateResource<Field>(Field(field_schema->GetName(), field_idx), field); status = store.CreateResource<Field>(Field(field_schema->GetName(), field_idx), field);
AddStep(*field); AddStepWithLsn(*field, c_context_.lsn);
MappingT element_ids = {}; MappingT element_ids = {};
FieldElementPtr raw_element; FieldElementPtr raw_element;
status = store.CreateResource<FieldElement>( status = store.CreateResource<FieldElement>(
FieldElement(collection->GetID(), field->GetID(), "RAW", FieldElementType::RAW), raw_element); FieldElement(collection->GetID(), field->GetID(), "RAW", FieldElementType::RAW), raw_element);
AddStep(*raw_element); AddStepWithLsn(*raw_element, c_context_.lsn);
element_ids.insert(raw_element->GetID()); element_ids.insert(raw_element->GetID());
for (auto& element_schema : field_elements) { for (auto& element_schema : field_elements) {
FieldElementPtr element; FieldElementPtr element;
...@@ -486,30 +488,33 @@ CreateCollectionOperation::DoExecute(Store& store) { ...@@ -486,30 +488,33 @@ CreateCollectionOperation::DoExecute(Store& store) {
store.CreateResource<FieldElement>(FieldElement(collection->GetID(), field->GetID(), store.CreateResource<FieldElement>(FieldElement(collection->GetID(), field->GetID(),
element_schema->GetName(), element_schema->GetFtype()), element_schema->GetName(), element_schema->GetFtype()),
element); element);
AddStep(*element); AddStepWithLsn(*element, c_context_.lsn);
element_ids.insert(element->GetID()); element_ids.insert(element->GetID());
} }
FieldCommitPtr field_commit; FieldCommitPtr field_commit;
status = store.CreateResource<FieldCommit>(FieldCommit(collection->GetID(), field->GetID(), element_ids), status = store.CreateResource<FieldCommit>(FieldCommit(collection->GetID(), field->GetID(), element_ids),
field_commit); field_commit);
AddStep(*field_commit); AddStepWithLsn(*field_commit, c_context_.lsn);
field_commit_ids.insert(field_commit->GetID()); field_commit_ids.insert(field_commit->GetID());
} }
SchemaCommitPtr schema_commit; SchemaCommitPtr schema_commit;
status = store.CreateResource<SchemaCommit>(SchemaCommit(collection->GetID(), field_commit_ids), schema_commit); status = store.CreateResource<SchemaCommit>(SchemaCommit(collection->GetID(), field_commit_ids), schema_commit);
AddStep(*schema_commit); AddStepWithLsn(*schema_commit, c_context_.lsn);
PartitionPtr partition; PartitionPtr partition;
status = store.CreateResource<Partition>(Partition("_default", collection->GetID()), partition); status = store.CreateResource<Partition>(Partition("_default", collection->GetID()), partition);
AddStep(*partition); AddStepWithLsn(*partition, c_context_.lsn);
context_.new_partition = partition;
PartitionCommitPtr partition_commit; PartitionCommitPtr partition_commit;
status = store.CreateResource<PartitionCommit>(PartitionCommit(collection->GetID(), partition->GetID()), status = store.CreateResource<PartitionCommit>(PartitionCommit(collection->GetID(), partition->GetID()),
partition_commit); partition_commit);
AddStep(*partition_commit); AddStepWithLsn(*partition_commit, c_context_.lsn);
context_.new_partition_commit = partition_commit;
CollectionCommitPtr collection_commit; CollectionCommitPtr collection_commit;
status = store.CreateResource<CollectionCommit>( status = store.CreateResource<CollectionCommit>(
CollectionCommit(collection->GetID(), schema_commit->GetID(), {partition_commit->GetID()}), collection_commit); CollectionCommit(collection->GetID(), schema_commit->GetID(), {partition_commit->GetID()}), collection_commit);
AddStep(*collection_commit); AddStepWithLsn(*collection_commit, c_context_.lsn);
context_.collection_commit = collection_commit; context_.new_collection_commit = collection_commit;
c_context_.collection_commit = collection_commit;
return Status::OK(); return Status::OK();
} }
...@@ -521,9 +526,9 @@ CreateCollectionOperation::GetSnapshot(ScopedSnapshotT& ss) const { ...@@ -521,9 +526,9 @@ CreateCollectionOperation::GetSnapshot(ScopedSnapshotT& ss) const {
status = IDSNotEmptyRequried(); status = IDSNotEmptyRequried();
if (!status.ok()) if (!status.ok())
return status; return status;
if (!context_.collection_commit) if (!c_context_.collection_commit)
return Status(SS_CONSTRAINT_CHECK_ERROR, "No Snapshot is available"); return Status(SS_CONSTRAINT_CHECK_ERROR, "No Snapshot is available");
status = Snapshots::GetInstance().GetSnapshot(ss, context_.collection_commit->GetCollectionId()); status = Snapshots::GetInstance().GetSnapshot(ss, c_context_.collection_commit->GetCollectionId());
return status; return status;
} }
...@@ -533,7 +538,7 @@ SoftDeleteCollectionOperation::DoExecute(Store& store) { ...@@ -533,7 +538,7 @@ SoftDeleteCollectionOperation::DoExecute(Store& store) {
return Status(SS_INVALID_CONTEX_ERROR, "Invalid Context"); return Status(SS_INVALID_CONTEX_ERROR, "Invalid Context");
} }
context_.collection->Deactivate(); context_.collection->Deactivate();
AddStep(*context_.collection); AddStepWithLsn(*context_.collection, context_.lsn);
return Status::OK(); return Status::OK();
} }
......
core/src/db/snapshot/CompoundOperations.h
浏览文件 @ 295105ff
...@@ -19,10 +19,25 @@ namespace milvus { ...@@ -19,10 +19,25 @@ namespace milvus {
namespace engine { namespace engine {
namespace snapshot { namespace snapshot {
class BuildOperation : public Operations { class CompoundBaseOperation : public Operations {
public: public:
using BaseT = Operations; using BaseT = Operations;
CompoundBaseOperation(const OperationContext& context, ScopedSnapshotT prev_ss);
CompoundBaseOperation(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id = 0);
std::string
GetRepr() const override;
Status
PreCheck() override;
};
class BuildOperation : public CompoundBaseOperation {
public:
using BaseT = CompoundBaseOperation;
static constexpr const char* Name = "BO";
BuildOperation(const OperationContext& context, ScopedSnapshotT prev_ss); BuildOperation(const OperationContext& context, ScopedSnapshotT prev_ss);
BuildOperation(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id = 0); BuildOperation(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id = 0);
...@@ -33,16 +48,19 @@ class BuildOperation : public Operations { ...@@ -33,16 +48,19 @@ class BuildOperation : public Operations {
CommitNewSegmentFile(const SegmentFileContext& context, SegmentFilePtr& created); CommitNewSegmentFile(const SegmentFileContext& context, SegmentFilePtr& created);
std::string std::string
OperationRepr() const override; GetName() const override {
return Name;
}
protected: protected:
Status Status
CheckSegmentStale(ScopedSnapshotT& latest_snapshot, ID_TYPE segment_id) const; CheckSegmentStale(ScopedSnapshotT& latest_snapshot, ID_TYPE segment_id) const;
}; };
class NewSegmentOperation : public Operations { class NewSegmentOperation : public CompoundBaseOperation {
public: public:
using BaseT = Operations; using BaseT = CompoundBaseOperation;
static constexpr const char* Name = "NSO";
NewSegmentOperation(const OperationContext& context, ScopedSnapshotT prev_ss); NewSegmentOperation(const OperationContext& context, ScopedSnapshotT prev_ss);
NewSegmentOperation(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id = 0); NewSegmentOperation(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id = 0);
...@@ -55,11 +73,17 @@ class NewSegmentOperation : public Operations { ...@@ -55,11 +73,17 @@ class NewSegmentOperation : public Operations {
Status Status
CommitNewSegmentFile(const SegmentFileContext& context, SegmentFilePtr& created); CommitNewSegmentFile(const SegmentFileContext& context, SegmentFilePtr& created);
std::string
GetName() const override {
return Name;
}
}; };
class MergeOperation : public Operations { class MergeOperation : public CompoundBaseOperation {
public: public:
using BaseT = Operations; using BaseT = CompoundBaseOperation;
static constexpr const char* Name = "MO";
MergeOperation(const OperationContext& context, ScopedSnapshotT prev_ss); MergeOperation(const OperationContext& context, ScopedSnapshotT prev_ss);
MergeOperation(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id = 0); MergeOperation(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id = 0);
...@@ -73,12 +97,16 @@ class MergeOperation : public Operations { ...@@ -73,12 +97,16 @@ class MergeOperation : public Operations {
CommitNewSegmentFile(const SegmentFileContext& context, SegmentFilePtr&); CommitNewSegmentFile(const SegmentFileContext& context, SegmentFilePtr&);
std::string std::string
OperationRepr() const override; GetName() const override {
return Name;
}
}; };
class CreateCollectionOperation : public Operations { class CreateCollectionOperation : public CompoundBaseOperation {
public: public:
using BaseT = Operations; using BaseT = CompoundBaseOperation;
static constexpr const char* Name = "CCO";
explicit CreateCollectionOperation(const CreateCollectionContext& context); explicit CreateCollectionOperation(const CreateCollectionContext& context);
Status Status
...@@ -87,13 +115,31 @@ class CreateCollectionOperation : public Operations { ...@@ -87,13 +115,31 @@ class CreateCollectionOperation : public Operations {
Status Status
GetSnapshot(ScopedSnapshotT& ss) const override; GetSnapshot(ScopedSnapshotT& ss) const override;
Status
PreCheck() override;
const LSN_TYPE&
GetContextLsn() const override {
return c_context_.lsn;
}
std::string
GetRepr() const override;
std::string
GetName() const override {
return Name;
}
private: private:
CreateCollectionContext context_; CreateCollectionContext c_context_;
}; };
class CreatePartitionOperation : public Operations { class CreatePartitionOperation : public CompoundBaseOperation {
public: public:
using BaseT = Operations; using BaseT = CompoundBaseOperation;
static constexpr const char* Name = "CPO";
CreatePartitionOperation(const OperationContext& context, ScopedSnapshotT prev_ss); CreatePartitionOperation(const OperationContext& context, ScopedSnapshotT prev_ss);
CreatePartitionOperation(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id = 0); CreatePartitionOperation(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id = 0);
...@@ -105,18 +151,37 @@ class CreatePartitionOperation : public Operations { ...@@ -105,18 +151,37 @@ class CreatePartitionOperation : public Operations {
Status Status
PreCheck() override; PreCheck() override;
std::string
GetName() const override {
return Name;
}
}; };
class DropPartitionOperation : public Operations { class DropPartitionOperation : public CompoundBaseOperation {
public: public:
using BaseT = Operations; using BaseT = CompoundBaseOperation;
static constexpr const char* Name = "DPO";
DropPartitionOperation(const PartitionContext& context, ScopedSnapshotT prev_ss); DropPartitionOperation(const PartitionContext& context, ScopedSnapshotT prev_ss);
Status Status
DoExecute(Store&) override; DoExecute(Store&) override;
const LSN_TYPE&
GetContextLsn() const override {
return c_context_.lsn;
}
std::string
GetRepr() const override;
std::string
GetName() const override {
return Name;
}
protected: protected:
PartitionContext context_; PartitionContext c_context_;
}; };
class GetSnapshotIDsOperation : public Operations { class GetSnapshotIDsOperation : public Operations {
...@@ -154,15 +219,23 @@ class GetCollectionIDsOperation : public Operations { ...@@ -154,15 +219,23 @@ class GetCollectionIDsOperation : public Operations {
IDS_TYPE ids_; IDS_TYPE ids_;
}; };
class SoftDeleteCollectionOperation : public Operations { class SoftDeleteCollectionOperation : public CompoundBaseOperation {
public: public:
using BaseT = Operations; using BaseT = CompoundBaseOperation;
explicit SoftDeleteCollectionOperation(const OperationContext& context) : BaseT(context, ScopedSnapshotT()) { static constexpr const char* Name = "DCO";
explicit SoftDeleteCollectionOperation(const OperationContext& context, ScopedSnapshotT prev_ss)
: BaseT(context, prev_ss) {
} }
Status Status
DoExecute(Store& store) override; DoExecute(Store& store) override;
std::string
GetName() const override {
return Name;
}
private: private:
ID_TYPE collection_id_; ID_TYPE collection_id_;
}; };
......
core/src/db/snapshot/Context.cpp 0 → 100644
浏览文件 @ 295105ff
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "db/snapshot/Context.h"
#include <sstream>
namespace milvus {
namespace engine {
namespace snapshot {
std::string
PartitionContext::ToString() const {
std::stringstream ss;
if (id != 0) {
ss << "PID=" << id;
} else if (name != "") {
ss << "PNAME=\"" << name << "\"";
}
return ss.str();
}
std::string
OperationContext::ToString() const {
std::stringstream ss;
if (new_collection_commit) {
ss << "N_CC=" << new_collection_commit->GetID();
}
if (new_collection) {
ss << ",N_CID=" << new_collection->GetID();
ss << ",N_CNAME=\"" << new_collection->GetName() << "\"";
}
if (stale_partition_commit) {
ss << ",S_PC=" << stale_partition_commit->GetID();
}
if (new_partition_commit) {
ss << ",N_PC=" << new_partition_commit->GetID();
}
if (new_partition) {
ss << ",N_PID=" << new_partition->GetID();
ss << ",N_PNAME=\"" << new_partition->GetName() << "\"";
}
if (new_segment_commit) {
ss << ",N_SC=" << new_segment_commit->GetID();
}
if (new_segment) {
ss << ",N_SE=" << new_segment->GetID();
}
if (stale_segments.size()) {
ss << ",S_SE=[";
bool first = true;
for (auto& f : stale_segments) {
if (!first) {
ss << ",";
}
ss << f->GetID();
first = false;
}
ss << "]";
}
if (stale_segment_file) {
ss << ",S_SF=" << stale_segment_file->GetID();
}
if (new_segment_files.size() > 0) {
ss << ",N_SF=[";
bool first = true;
for (auto& f : new_segment_files) {
if (!first) {
ss << ",";
}
ss << f->GetID();
first = false;
}
ss << "]";
}
return ss.str();
}
std::string
CreateCollectionContext::ToString() const {
std::stringstream ss;
if (collection) {
ss << "CID=" << collection->GetID();
ss << ",CNAME=\"" << collection->GetName() << "\"";
}
return ss.str();
}
} // namespace snapshot
} // namespace engine
} // namespace milvus
core/src/db/snapshot/Context.h
浏览文件 @ 295105ff
...@@ -25,6 +25,10 @@ namespace snapshot { ...@@ -25,6 +25,10 @@ namespace snapshot {
struct PartitionContext { struct PartitionContext {
std::string name; std::string name;
ID_TYPE id = 0; ID_TYPE id = 0;
LSN_TYPE lsn = 0;
std::string
ToString() const;
}; };
struct SegmentFileContext { struct SegmentFileContext {
...@@ -46,6 +50,8 @@ struct OperationContext { ...@@ -46,6 +50,8 @@ struct OperationContext {
PartitionPtr new_partition = nullptr; PartitionPtr new_partition = nullptr;
PartitionCommitPtr new_partition_commit = nullptr; PartitionCommitPtr new_partition_commit = nullptr;
SchemaCommitPtr new_schema_commit = nullptr; SchemaCommitPtr new_schema_commit = nullptr;
CollectionCommitPtr new_collection_commit = nullptr;
CollectionPtr new_collection = nullptr;
SegmentFilePtr stale_segment_file = nullptr; SegmentFilePtr stale_segment_file = nullptr;
std::vector<SegmentPtr> stale_segments; std::vector<SegmentPtr> stale_segments;
...@@ -62,12 +68,20 @@ struct OperationContext { ...@@ -62,12 +68,20 @@ struct OperationContext {
SegmentFile::VecT new_segment_files; SegmentFile::VecT new_segment_files;
CollectionPtr collection = nullptr; CollectionPtr collection = nullptr;
LSN_TYPE lsn = 0;
std::string
ToString() const;
}; };
struct CreateCollectionContext { struct CreateCollectionContext {
CollectionPtr collection = nullptr; CollectionPtr collection = nullptr;
std::map<FieldPtr, std::vector<FieldElementPtr>> fields_schema; std::map<FieldPtr, std::vector<FieldElementPtr>> fields_schema;
CollectionCommitPtr collection_commit = nullptr; CollectionCommitPtr collection_commit = nullptr;
LSN_TYPE lsn = 0;
std::string
ToString() const;
}; };
} // namespace snapshot } // namespace snapshot
......
core/src/db/snapshot/Operations.cpp
浏览文件 @ 295105ff
...@@ -21,12 +21,23 @@ namespace snapshot { ...@@ -21,12 +21,23 @@ namespace snapshot {
static ID_TYPE UID = 1; static ID_TYPE UID = 1;
Operations::Operations(const OperationContext& context, ScopedSnapshotT prev_ss) std::ostream&
: context_(context), prev_ss_(prev_ss), uid_(UID++), status_(40005, "Operation Pending") { operator<<(std::ostream& out, const Operations& operation) {
out << operation.ToString();
return out;
} }
Operations::Operations(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id) Operations::Operations(const OperationContext& context, ScopedSnapshotT prev_ss, const OperationsType& type)
: context_(context), uid_(UID++), status_(40005, "Operation Pending") { : context_(context),
prev_ss_(prev_ss),
uid_(UID++),
status_(SS_OPERATION_PENDING, "Operation Pending"),
type_(type) {
}
Operations::Operations(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id,
const OperationsType& type)
: context_(context), uid_(UID++), status_(SS_OPERATION_PENDING, "Operation Pending"), type_(type) {
auto status = Snapshots::GetInstance().GetSnapshot(prev_ss_, collection_id, commit_id); auto status = Snapshots::GetInstance().GetSnapshot(prev_ss_, collection_id, commit_id);
if (!status.ok()) if (!status.ok())
prev_ss_ = ScopedSnapshotT(); prev_ss_ = ScopedSnapshotT();
...@@ -43,16 +54,16 @@ Operations::FailureString() const { ...@@ -43,16 +54,16 @@ Operations::FailureString() const {
} }
std::string std::string
Operations::OperationRepr() const { Operations::GetRepr() const {
std::stringstream ss; std::stringstream ss;
ss << "<" << OperationName() << ":" << GetID() << ">"; ss << "<" << GetName() << ":" << GetID() << ">";
return ss.str(); return ss.str();
} }
std::string std::string
Operations::ToString() const { Operations::ToString() const {
std::stringstream ss; std::stringstream ss;
ss << OperationRepr(); ss << GetRepr();
ss << (done_ ? " | DONE" : " | PENDING"); ss << (done_ ? " | DONE" : " | PENDING");
if (done_) { if (done_) {
if (status_.ok()) { if (status_.ok()) {
...@@ -71,6 +82,9 @@ Operations::GetID() const { ...@@ -71,6 +82,9 @@ Operations::GetID() const {
Status Status
Operations::operator()(Store& store) { Operations::operator()(Store& store) {
auto status = PreCheck();
if (!status.ok())
return status;
return ApplyToStore(store); return ApplyToStore(store);
} }
...@@ -90,7 +104,9 @@ void ...@@ -90,7 +104,9 @@ void
Operations::Done() { Operations::Done() {
std::unique_lock<std::mutex> lock(finish_mtx_); std::unique_lock<std::mutex> lock(finish_mtx_);
done_ = true; done_ = true;
std::cout << ToString() << std::endl; if (GetType() == OperationsType::W_Compound) {
std::cout << ToString() << std::endl;
}
finish_cond_.notify_all(); finish_cond_.notify_all();
} }
...@@ -171,7 +187,9 @@ Operations::GetSnapshot(ScopedSnapshotT& ss) const { ...@@ -171,7 +187,9 @@ Operations::GetSnapshot(ScopedSnapshotT& ss) const {
Status Status
Operations::ApplyToStore(Store& store) { Operations::ApplyToStore(Store& store) {
std::cout << ToString() << std::endl; if (GetType() == OperationsType::W_Compound) {
std::cout << ToString() << std::endl;
}
if (done_) { if (done_) {
Done(); Done();
return status_; return status_;
...@@ -210,6 +228,23 @@ Operations::PostExecute(Store& store) { ...@@ -210,6 +228,23 @@ Operations::PostExecute(Store& store) {
return store.DoCommitOperation(*this); return store.DoCommitOperation(*this);
} }
Status
Operations::RollBack() {
// TODO: Implement here
// Spwarn a rollback operation or re-use this operation
return Status::OK();
}
Status
Operations::ApplyRollBack(Store& store) {
// TODO: Implement rollback to remove all resources in steps_
return Status::OK();
}
Operations::~Operations() {
// TODO: Prefer to submit a rollback operation if status is not ok
}
} // namespace snapshot } // namespace snapshot
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
core/src/db/snapshot/Operations.h
浏览文件 @ 295105ff
...@@ -33,16 +33,25 @@ namespace snapshot { ...@@ -33,16 +33,25 @@ namespace snapshot {
using StepsT = std::vector<std::any>; using StepsT = std::vector<std::any>;
using CheckStaleFunc = std::function<Status(ScopedSnapshotT&)>; using CheckStaleFunc = std::function<Status(ScopedSnapshotT&)>;
enum OperationsType { Invalid, W_Leaf, O_Leaf, W_Compound, O_Compound };
class Operations : public std::enable_shared_from_this<Operations> { class Operations : public std::enable_shared_from_this<Operations> {
public: public:
Operations(const OperationContext& context, ScopedSnapshotT prev_ss); Operations(const OperationContext& context, ScopedSnapshotT prev_ss,
Operations(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id = 0); const OperationsType& type = OperationsType::Invalid);
Operations(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id = 0,
const OperationsType& type = OperationsType::Invalid);
const ScopedSnapshotT& const ScopedSnapshotT&
GetPrevSnapshot() const { GetPrevSnapshot() const {
return prev_ss_; return prev_ss_;
} }
virtual const LSN_TYPE&
GetContextLsn() const {
return context_.lsn;
}
virtual Status virtual Status
CheckStale(const CheckStaleFunc& checker = nullptr) const; CheckStale(const CheckStaleFunc& checker = nullptr) const;
virtual Status virtual Status
...@@ -51,6 +60,9 @@ class Operations : public std::enable_shared_from_this<Operations> { ...@@ -51,6 +60,9 @@ class Operations : public std::enable_shared_from_this<Operations> {
template <typename StepT> template <typename StepT>
void void
AddStep(const StepT& step, bool activate = true); AddStep(const StepT& step, bool activate = true);
template <typename StepT>
void
AddStepWithLsn(const StepT& step, const LSN_TYPE& lsn, bool activate = true);
void void
SetStepResult(ID_TYPE id) { SetStepResult(ID_TYPE id) {
ids_.push_back(id); ids_.push_back(id);
...@@ -64,6 +76,11 @@ class Operations : public std::enable_shared_from_this<Operations> { ...@@ -64,6 +76,11 @@ class Operations : public std::enable_shared_from_this<Operations> {
ID_TYPE ID_TYPE
GetID() const; GetID() const;
virtual const OperationsType&
GetType() const {
return type_;
}
virtual Status virtual Status
OnExecute(Store&); OnExecute(Store&);
virtual Status virtual Status
...@@ -101,18 +118,23 @@ class Operations : public std::enable_shared_from_this<Operations> { ...@@ -101,18 +118,23 @@ class Operations : public std::enable_shared_from_this<Operations> {
return status_; return status_;
} }
std::string virtual std::string
OperationName() const { GetName() const {
return typeid(*this).name(); return typeid(*this).name();
} }
virtual std::string virtual std::string
OperationRepr() const; GetRepr() const;
virtual std::string virtual std::string
ToString() const; ToString() const;
virtual ~Operations() { Status
} RollBack();
virtual ~Operations();
friend std::ostream&
operator<<(std::ostream& out, const Operations& operation);
protected: protected:
virtual std::string virtual std::string
...@@ -127,6 +149,9 @@ class Operations : public std::enable_shared_from_this<Operations> { ...@@ -127,6 +149,9 @@ class Operations : public std::enable_shared_from_this<Operations> {
Status Status
PrevSnapshotRequried() const; PrevSnapshotRequried() const;
Status
ApplyRollBack(Store&);
OperationContext context_; OperationContext context_;
ScopedSnapshotT prev_ss_; ScopedSnapshotT prev_ss_;
StepsT steps_; StepsT steps_;
...@@ -136,6 +161,7 @@ class Operations : public std::enable_shared_from_this<Operations> { ...@@ -136,6 +161,7 @@ class Operations : public std::enable_shared_from_this<Operations> {
mutable std::mutex finish_mtx_; mutable std::mutex finish_mtx_;
std::condition_variable finish_cond_; std::condition_variable finish_cond_;
ID_TYPE uid_; ID_TYPE uid_;
OperationsType type_;
}; };
template <typename StepT> template <typename StepT>
...@@ -147,14 +173,25 @@ Operations::AddStep(const StepT& step, bool activate) { ...@@ -147,14 +173,25 @@ Operations::AddStep(const StepT& step, bool activate) {
steps_.push_back(s); steps_.push_back(s);
} }
template <typename StepT>
void
Operations::AddStepWithLsn(const StepT& step, const LSN_TYPE& lsn, bool activate) {
auto s = std::make_shared<StepT>(step);
if (activate)
s->Activate();
s->SetLsn(lsn);
steps_.push_back(s);
}
template <typename ResourceT> template <typename ResourceT>
class CommitOperation : public Operations { class CommitOperation : public Operations {
public: public:
using BaseT = Operations; using BaseT = Operations;
CommitOperation(const OperationContext& context, ScopedSnapshotT prev_ss) : BaseT(context, prev_ss) { CommitOperation(const OperationContext& context, ScopedSnapshotT prev_ss)
: BaseT(context, prev_ss, OperationsType::W_Leaf) {
} }
CommitOperation(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id = 0) CommitOperation(const OperationContext& context, ID_TYPE collection_id, ID_TYPE commit_id = 0)
: BaseT(context, collection_id, commit_id) { : BaseT(context, collection_id, commit_id, OperationsType::W_Leaf) {
} }
virtual typename ResourceT::Ptr virtual typename ResourceT::Ptr
...@@ -194,7 +231,7 @@ template <typename ResourceT> ...@@ -194,7 +231,7 @@ template <typename ResourceT>
class LoadOperation : public Operations { class LoadOperation : public Operations {
public: public:
explicit LoadOperation(const LoadOperationContext& context) explicit LoadOperation(const LoadOperationContext& context)
: Operations(OperationContext(), ScopedSnapshotT()), context_(context) { : Operations(OperationContext(), ScopedSnapshotT(), OperationsType::O_Leaf), context_(context) {
} }
Status Status
...@@ -240,7 +277,8 @@ class LoadOperation : public Operations { ...@@ -240,7 +277,8 @@ class LoadOperation : public Operations {
template <typename ResourceT> template <typename ResourceT>
class HardDeleteOperation : public Operations { class HardDeleteOperation : public Operations {
public: public:
explicit HardDeleteOperation(ID_TYPE id) : Operations(OperationContext(), ScopedSnapshotT()), id_(id) { explicit HardDeleteOperation(ID_TYPE id)
: Operations(OperationContext(), ScopedSnapshotT(), OperationsType::W_Leaf), id_(id) {
} }
Status Status
...@@ -260,7 +298,8 @@ class HardDeleteOperation : public Operations { ...@@ -260,7 +298,8 @@ class HardDeleteOperation : public Operations {
template <> template <>
class HardDeleteOperation<Collection> : public Operations { class HardDeleteOperation<Collection> : public Operations {
public: public:
explicit HardDeleteOperation(ID_TYPE id) : Operations(OperationContext(), ScopedSnapshotT()), id_(id) { explicit HardDeleteOperation(ID_TYPE id)
: Operations(OperationContext(), ScopedSnapshotT(), OperationsType::W_Leaf), id_(id) {
} }
Status Status
......
core/src/db/snapshot/ResourceOperations.h
浏览文件 @ 295105ff
...@@ -113,7 +113,7 @@ template <> ...@@ -113,7 +113,7 @@ template <>
class LoadOperation<Collection> : public Operations { class LoadOperation<Collection> : public Operations {
public: public:
explicit LoadOperation(const LoadOperationContext& context) explicit LoadOperation(const LoadOperationContext& context)
: Operations(OperationContext(), ScopedSnapshotT()), context_(context) { : Operations(OperationContext(), ScopedSnapshotT(), OperationsType::O_Leaf), context_(context) {
} }
Status Status
......
core/src/db/snapshot/ResourceTypes.h
浏览文件 @ 295105ff
...@@ -23,6 +23,7 @@ using ID_TYPE = int64_t; ...@@ -23,6 +23,7 @@ using ID_TYPE = int64_t;
using NUM_TYPE = int64_t; using NUM_TYPE = int64_t;
using FTYPE_TYPE = int64_t; using FTYPE_TYPE = int64_t;
using TS_TYPE = int64_t; using TS_TYPE = int64_t;
using LSN_TYPE = uint64_t;
using MappingT = std::set<ID_TYPE>; using MappingT = std::set<ID_TYPE>;
enum FieldElementType { RAW, IVFSQ8 }; enum FieldElementType { RAW, IVFSQ8 };
......
core/src/db/snapshot/Resources.cpp
浏览文件 @ 295105ff
...@@ -18,93 +18,93 @@ namespace milvus { ...@@ -18,93 +18,93 @@ namespace milvus {
namespace engine { namespace engine {
namespace snapshot { namespace snapshot {
Collection::Collection(const std::string& name, ID_TYPE id, State status, TS_TYPE created_on, TS_TYPE updated_on) Collection::Collection(const std::string& name, ID_TYPE id, LSN_TYPE lsn, State status, TS_TYPE created_on,
: NameField(name), IdField(id), StatusField(status), CreatedOnField(created_on), UpdatedOnField(updated_on) { TS_TYPE updated_on)
: NameField(name),
IdField(id),
LsnField(lsn),
StatusField(status),
CreatedOnField(created_on),
UpdatedOnField(updated_on) {
} }
SchemaCommit::SchemaCommit(ID_TYPE collection_id, const MappingT& mappings, ID_TYPE id, State status, SchemaCommit::SchemaCommit(ID_TYPE collection_id, const MappingT& mappings, ID_TYPE id, LSN_TYPE lsn, State status,
TS_TYPE created_on, TS_TYPE updated_on) TS_TYPE created_on, TS_TYPE updated_on)
: CollectionIdField(collection_id), : CollectionIdField(collection_id),
MappingsField(mappings), MappingsField(mappings),
IdField(id), IdField(id),
LsnField(lsn),
StatusField(status), StatusField(status),
CreatedOnField(created_on), CreatedOnField(created_on),
UpdatedOnField(updated_on) { UpdatedOnField(updated_on) {
} }
FieldCommit::FieldCommit(ID_TYPE collection_id, ID_TYPE field_id, const MappingT& mappings, ID_TYPE id, State status, FieldCommit::FieldCommit(ID_TYPE collection_id, ID_TYPE field_id, const MappingT& mappings, ID_TYPE id, LSN_TYPE lsn,
TS_TYPE created_on, TS_TYPE updated_on) State status, TS_TYPE created_on, TS_TYPE updated_on)
: CollectionIdField(collection_id), : CollectionIdField(collection_id),
FieldIdField(field_id), FieldIdField(field_id),
MappingsField(mappings), MappingsField(mappings),
IdField(id), IdField(id),
LsnField(lsn),
StatusField(status), StatusField(status),
CreatedOnField(created_on), CreatedOnField(created_on),
UpdatedOnField(updated_on) { UpdatedOnField(updated_on) {
} }
Field::Field(const std::string& name, NUM_TYPE num, ID_TYPE id, State status, TS_TYPE created_on, TS_TYPE updated_on) Field::Field(const std::string& name, NUM_TYPE num, ID_TYPE id, LSN_TYPE lsn, State status, TS_TYPE created_on,
TS_TYPE updated_on)
: NameField(name), : NameField(name),
NumField(num), NumField(num),
IdField(id), IdField(id),
LsnField(lsn),
StatusField(status), StatusField(status),
CreatedOnField(created_on), CreatedOnField(created_on),
UpdatedOnField(updated_on_) { UpdatedOnField(updated_on_) {
} }
FieldElement::FieldElement(ID_TYPE collection_id, ID_TYPE field_id, const std::string& name, FTYPE_TYPE ftype, FieldElement::FieldElement(ID_TYPE collection_id, ID_TYPE field_id, const std::string& name, FTYPE_TYPE ftype,
ID_TYPE id, State status, TS_TYPE created_on, TS_TYPE updated_on) ID_TYPE id, LSN_TYPE lsn, State status, TS_TYPE created_on, TS_TYPE updated_on)
: CollectionIdField(collection_id), : CollectionIdField(collection_id),
FieldIdField(field_id), FieldIdField(field_id),
NameField(name), NameField(name),
FtypeField(ftype), FtypeField(ftype),
IdField(id), IdField(id),
LsnField(lsn),
StatusField(status), StatusField(status),
CreatedOnField(created_on), CreatedOnField(created_on),
UpdatedOnField(updated_on) { UpdatedOnField(updated_on) {
} }
CollectionCommit::CollectionCommit(ID_TYPE collection_id, ID_TYPE schema_id, const MappingT& mappings, ID_TYPE id, CollectionCommit::CollectionCommit(ID_TYPE collection_id, ID_TYPE schema_id, const MappingT& mappings, ID_TYPE id,
State status, TS_TYPE created_on, TS_TYPE updated_on) LSN_TYPE lsn, State status, TS_TYPE created_on, TS_TYPE updated_on)
: CollectionIdField(collection_id), : CollectionIdField(collection_id),
SchemaIdField(schema_id), SchemaIdField(schema_id),
MappingsField(mappings), MappingsField(mappings),
IdField(id), IdField(id),
LsnField(lsn),
StatusField(status), StatusField(status),
CreatedOnField(created_on), CreatedOnField(created_on),
UpdatedOnField(updated_on) { UpdatedOnField(updated_on) {
} }
/* std::string CollectionCommit::ToString() const { */ Partition::Partition(const std::string& name, ID_TYPE collection_id, ID_TYPE id, LSN_TYPE lsn, State status,
/* std::stringstream ss; */ TS_TYPE created_on, TS_TYPE updated_on)
/* ss << "<" << BaseT::ToString() << ", Mappings=" << "["; */
/* bool first = true; */
/* std::string prefix; */
/* for (auto& id : mappings_) { */
/* if (!first) prefix = ", "; */
/* else first = false; */
/* ss << prefix << id; */
/* } */
/* ss << "]>"; */
/* return ss.str(); */
/* } */
Partition::Partition(const std::string& name, ID_TYPE collection_id, ID_TYPE id, State status, TS_TYPE created_on,
TS_TYPE updated_on)
: NameField(name), : NameField(name),
CollectionIdField(collection_id), CollectionIdField(collection_id),
IdField(id), IdField(id),
LsnField(lsn),
StatusField(status), StatusField(status),
CreatedOnField(created_on), CreatedOnField(created_on),
UpdatedOnField(updated_on) { UpdatedOnField(updated_on) {
} }
PartitionCommit::PartitionCommit(ID_TYPE collection_id, ID_TYPE partition_id, const MappingT& mappings, ID_TYPE id, PartitionCommit::PartitionCommit(ID_TYPE collection_id, ID_TYPE partition_id, const MappingT& mappings, ID_TYPE id,
State status, TS_TYPE created_on, TS_TYPE updated_on) LSN_TYPE lsn, State status, TS_TYPE created_on, TS_TYPE updated_on)
: CollectionIdField(collection_id), : CollectionIdField(collection_id),
PartitionIdField(partition_id), PartitionIdField(partition_id),
MappingsField(mappings), MappingsField(mappings),
IdField(id), IdField(id),
LsnField(lsn),
StatusField(status), StatusField(status),
CreatedOnField(created_on), CreatedOnField(created_on),
UpdatedOnField(updated_on) { UpdatedOnField(updated_on) {
...@@ -123,10 +123,12 @@ PartitionCommit::ToString() const { ...@@ -123,10 +123,12 @@ PartitionCommit::ToString() const {
return ss.str(); return ss.str();
} }
Segment::Segment(ID_TYPE partition_id, ID_TYPE num, ID_TYPE id, State status, TS_TYPE created_on, TS_TYPE updated_on) Segment::Segment(ID_TYPE partition_id, ID_TYPE num, ID_TYPE id, LSN_TYPE lsn, State status, TS_TYPE created_on,
TS_TYPE updated_on)
: PartitionIdField(partition_id), : PartitionIdField(partition_id),
NumField(num), NumField(num),
IdField(id), IdField(id),
LsnField(lsn),
StatusField(status), StatusField(status),
CreatedOnField(created_on), CreatedOnField(created_on),
UpdatedOnField(updated_on) { UpdatedOnField(updated_on) {
...@@ -144,12 +146,13 @@ Segment::ToString() const { ...@@ -144,12 +146,13 @@ Segment::ToString() const {
} }
SegmentCommit::SegmentCommit(ID_TYPE schema_id, ID_TYPE partition_id, ID_TYPE segment_id, const MappingT& mappings, SegmentCommit::SegmentCommit(ID_TYPE schema_id, ID_TYPE partition_id, ID_TYPE segment_id, const MappingT& mappings,
ID_TYPE id, State status, TS_TYPE created_on, TS_TYPE updated_on) ID_TYPE id, LSN_TYPE lsn, State status, TS_TYPE created_on, TS_TYPE updated_on)
: SchemaIdField(schema_id), : SchemaIdField(schema_id),
PartitionIdField(partition_id), PartitionIdField(partition_id),
SegmentIdField(segment_id), SegmentIdField(segment_id),
MappingsField(mappings), MappingsField(mappings),
IdField(id), IdField(id),
LsnField(lsn),
StatusField(status), StatusField(status),
CreatedOnField(created_on), CreatedOnField(created_on),
UpdatedOnField(updated_on) { UpdatedOnField(updated_on) {
...@@ -166,12 +169,13 @@ SegmentCommit::ToString() const { ...@@ -166,12 +169,13 @@ SegmentCommit::ToString() const {
return ss.str(); return ss.str();
} }
SegmentFile::SegmentFile(ID_TYPE partition_id, ID_TYPE segment_id, ID_TYPE field_element_id, ID_TYPE id, State status, SegmentFile::SegmentFile(ID_TYPE partition_id, ID_TYPE segment_id, ID_TYPE field_element_id, ID_TYPE id, LSN_TYPE lsn,
TS_TYPE created_on, TS_TYPE updated_on) State status, TS_TYPE created_on, TS_TYPE updated_on)
: PartitionIdField(partition_id), : PartitionIdField(partition_id),
SegmentIdField(segment_id), SegmentIdField(segment_id),
FieldElementIdField(field_element_id), FieldElementIdField(field_element_id),
IdField(id), IdField(id),
LsnField(lsn),
StatusField(status), StatusField(status),
CreatedOnField(created_on), CreatedOnField(created_on),
UpdatedOnField(updated_on) { UpdatedOnField(updated_on) {
......
core/src/db/snapshot/Resources.h
浏览文件 @ 295105ff
...@@ -86,6 +86,24 @@ class StatusField { ...@@ -86,6 +86,24 @@ class StatusField {
State status_; State status_;
}; };
class LsnField {
public:
explicit LsnField(LSN_TYPE lsn = 0) : lsn_(lsn) {
}
const LSN_TYPE&
GetLsn() const {
return lsn_;
}
void
SetLsn(const LSN_TYPE& lsn) {
lsn_ = lsn;
}
protected:
LSN_TYPE lsn_;
};
class CreatedOnField { class CreatedOnField {
public: public:
explicit CreatedOnField(TS_TYPE created_on = GetMicroSecTimeStamp()) : created_on_(created_on) { explicit CreatedOnField(TS_TYPE created_on = GetMicroSecTimeStamp()) : created_on_(created_on) {
...@@ -261,6 +279,7 @@ class NameField { ...@@ -261,6 +279,7 @@ class NameField {
class Collection : public DBBaseResource<>, class Collection : public DBBaseResource<>,
public NameField, public NameField,
public IdField, public IdField,
public LsnField,
public StatusField, public StatusField,
public CreatedOnField, public CreatedOnField,
public UpdatedOnField { public UpdatedOnField {
...@@ -271,7 +290,7 @@ class Collection : public DBBaseResource<>, ...@@ -271,7 +290,7 @@ class Collection : public DBBaseResource<>,
using VecT = std::vector<Ptr>; using VecT = std::vector<Ptr>;
static constexpr const char* Name = "Collection"; static constexpr const char* Name = "Collection";
Collection(const std::string& name, ID_TYPE id = 0, State status = PENDING, Collection(const std::string& name, ID_TYPE id = 0, LSN_TYPE lsn = 0, State status = PENDING,
TS_TYPE created_on = GetMicroSecTimeStamp(), TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); TS_TYPE created_on = GetMicroSecTimeStamp(), TS_TYPE UpdatedOnField = GetMicroSecTimeStamp());
}; };
...@@ -281,6 +300,7 @@ class SchemaCommit : public DBBaseResource<>, ...@@ -281,6 +300,7 @@ class SchemaCommit : public DBBaseResource<>,
public CollectionIdField, public CollectionIdField,
public MappingsField, public MappingsField,
public IdField, public IdField,
public LsnField,
public StatusField, public StatusField,
public CreatedOnField, public CreatedOnField,
public UpdatedOnField { public UpdatedOnField {
...@@ -291,8 +311,9 @@ class SchemaCommit : public DBBaseResource<>, ...@@ -291,8 +311,9 @@ class SchemaCommit : public DBBaseResource<>,
using VecT = std::vector<Ptr>; using VecT = std::vector<Ptr>;
static constexpr const char* Name = "SchemaCommit"; static constexpr const char* Name = "SchemaCommit";
SchemaCommit(ID_TYPE collection_id, const MappingT& mappings = {}, ID_TYPE id = 0, State status = PENDING, SchemaCommit(ID_TYPE collection_id, const MappingT& mappings = {}, ID_TYPE id = 0, LSN_TYPE lsn = 0,
TS_TYPE created_on = GetMicroSecTimeStamp(), TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(),
TS_TYPE UpdatedOnField = GetMicroSecTimeStamp());
}; };
using SchemaCommitPtr = SchemaCommit::Ptr; using SchemaCommitPtr = SchemaCommit::Ptr;
...@@ -301,6 +322,7 @@ class Field : public DBBaseResource<>, ...@@ -301,6 +322,7 @@ class Field : public DBBaseResource<>,
public NameField, public NameField,
public NumField, public NumField,
public IdField, public IdField,
public LsnField,
public StatusField, public StatusField,
public CreatedOnField, public CreatedOnField,
public UpdatedOnField { public UpdatedOnField {
...@@ -311,7 +333,7 @@ class Field : public DBBaseResource<>, ...@@ -311,7 +333,7 @@ class Field : public DBBaseResource<>,
using VecT = std::vector<Ptr>; using VecT = std::vector<Ptr>;
static constexpr const char* Name = "Field"; static constexpr const char* Name = "Field";
Field(const std::string& name, NUM_TYPE num, ID_TYPE id = 0, State status = PENDING, Field(const std::string& name, NUM_TYPE num, ID_TYPE id = 0, LSN_TYPE lsn = 0, State status = PENDING,
TS_TYPE created_on = GetMicroSecTimeStamp(), TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); TS_TYPE created_on = GetMicroSecTimeStamp(), TS_TYPE UpdatedOnField = GetMicroSecTimeStamp());
}; };
...@@ -322,6 +344,7 @@ class FieldCommit : public DBBaseResource<>, ...@@ -322,6 +344,7 @@ class FieldCommit : public DBBaseResource<>,
public FieldIdField, public FieldIdField,
public MappingsField, public MappingsField,
public IdField, public IdField,
public LsnField,
public StatusField, public StatusField,
public CreatedOnField, public CreatedOnField,
public UpdatedOnField { public UpdatedOnField {
...@@ -333,7 +356,7 @@ class FieldCommit : public DBBaseResource<>, ...@@ -333,7 +356,7 @@ class FieldCommit : public DBBaseResource<>,
static constexpr const char* Name = "FieldCommit"; static constexpr const char* Name = "FieldCommit";
FieldCommit(ID_TYPE collection_id, ID_TYPE field_id, const MappingT& mappings = {}, ID_TYPE id = 0, FieldCommit(ID_TYPE collection_id, ID_TYPE field_id, const MappingT& mappings = {}, ID_TYPE id = 0,
State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(), LSN_TYPE lsn = 0, State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(),
TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); TS_TYPE UpdatedOnField = GetMicroSecTimeStamp());
}; };
...@@ -345,6 +368,7 @@ class FieldElement : public DBBaseResource<>, ...@@ -345,6 +368,7 @@ class FieldElement : public DBBaseResource<>,
public NameField, public NameField,
public FtypeField, public FtypeField,
public IdField, public IdField,
public LsnField,
public StatusField, public StatusField,
public CreatedOnField, public CreatedOnField,
public UpdatedOnField { public UpdatedOnField {
...@@ -355,7 +379,7 @@ class FieldElement : public DBBaseResource<>, ...@@ -355,7 +379,7 @@ class FieldElement : public DBBaseResource<>,
using VecT = std::vector<Ptr>; using VecT = std::vector<Ptr>;
static constexpr const char* Name = "FieldElement"; static constexpr const char* Name = "FieldElement";
FieldElement(ID_TYPE collection_id, ID_TYPE field_id, const std::string& name, FTYPE_TYPE ftype, ID_TYPE id = 0, FieldElement(ID_TYPE collection_id, ID_TYPE field_id, const std::string& name, FTYPE_TYPE ftype, ID_TYPE id = 0,
State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(), LSN_TYPE lsn = 0, State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(),
TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); TS_TYPE UpdatedOnField = GetMicroSecTimeStamp());
}; };
...@@ -366,6 +390,7 @@ class CollectionCommit : public DBBaseResource<>, ...@@ -366,6 +390,7 @@ class CollectionCommit : public DBBaseResource<>,
public SchemaIdField, public SchemaIdField,
public MappingsField, public MappingsField,
public IdField, public IdField,
public LsnField,
public StatusField, public StatusField,
public CreatedOnField, public CreatedOnField,
public UpdatedOnField { public UpdatedOnField {
...@@ -376,7 +401,7 @@ class CollectionCommit : public DBBaseResource<>, ...@@ -376,7 +401,7 @@ class CollectionCommit : public DBBaseResource<>,
using ScopedMapT = std::map<ID_TYPE, ScopedResource<CollectionCommit>>; using ScopedMapT = std::map<ID_TYPE, ScopedResource<CollectionCommit>>;
using VecT = std::vector<Ptr>; using VecT = std::vector<Ptr>;
CollectionCommit(ID_TYPE collection_id, ID_TYPE schema_id, const MappingT& mappings = {}, ID_TYPE id = 0, CollectionCommit(ID_TYPE collection_id, ID_TYPE schema_id, const MappingT& mappings = {}, ID_TYPE id = 0,
State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(), LSN_TYPE lsn = 0, State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(),
TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); TS_TYPE UpdatedOnField = GetMicroSecTimeStamp());
}; };
...@@ -386,6 +411,7 @@ class Partition : public DBBaseResource<>, ...@@ -386,6 +411,7 @@ class Partition : public DBBaseResource<>,
public NameField, public NameField,
public CollectionIdField, public CollectionIdField,
public IdField, public IdField,
public LsnField,
public StatusField, public StatusField,
public CreatedOnField, public CreatedOnField,
public UpdatedOnField { public UpdatedOnField {
...@@ -396,7 +422,7 @@ class Partition : public DBBaseResource<>, ...@@ -396,7 +422,7 @@ class Partition : public DBBaseResource<>,
using VecT = std::vector<Ptr>; using VecT = std::vector<Ptr>;
static constexpr const char* Name = "Partition"; static constexpr const char* Name = "Partition";
Partition(const std::string& name, ID_TYPE collection_id, ID_TYPE id = 0, State status = PENDING, Partition(const std::string& name, ID_TYPE collection_id, ID_TYPE id = 0, LSN_TYPE lsn = 0, State status = PENDING,
TS_TYPE created_on = GetMicroSecTimeStamp(), TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); TS_TYPE created_on = GetMicroSecTimeStamp(), TS_TYPE UpdatedOnField = GetMicroSecTimeStamp());
}; };
...@@ -407,6 +433,7 @@ class PartitionCommit : public DBBaseResource<>, ...@@ -407,6 +433,7 @@ class PartitionCommit : public DBBaseResource<>,
public PartitionIdField, public PartitionIdField,
public MappingsField, public MappingsField,
public IdField, public IdField,
public LsnField,
public StatusField, public StatusField,
public CreatedOnField, public CreatedOnField,
public UpdatedOnField { public UpdatedOnField {
...@@ -418,7 +445,7 @@ class PartitionCommit : public DBBaseResource<>, ...@@ -418,7 +445,7 @@ class PartitionCommit : public DBBaseResource<>,
static constexpr const char* Name = "PartitionCommit"; static constexpr const char* Name = "PartitionCommit";
PartitionCommit(ID_TYPE collection_id, ID_TYPE partition_id, const MappingT& mappings = {}, ID_TYPE id = 0, PartitionCommit(ID_TYPE collection_id, ID_TYPE partition_id, const MappingT& mappings = {}, ID_TYPE id = 0,
State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(), LSN_TYPE lsn = 0, State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(),
TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); TS_TYPE UpdatedOnField = GetMicroSecTimeStamp());
std::string std::string
...@@ -431,6 +458,7 @@ class Segment : public DBBaseResource<>, ...@@ -431,6 +458,7 @@ class Segment : public DBBaseResource<>,
public PartitionIdField, public PartitionIdField,
public NumField, public NumField,
public IdField, public IdField,
public LsnField,
public StatusField, public StatusField,
public CreatedOnField, public CreatedOnField,
public UpdatedOnField { public UpdatedOnField {
...@@ -441,7 +469,7 @@ class Segment : public DBBaseResource<>, ...@@ -441,7 +469,7 @@ class Segment : public DBBaseResource<>,
using VecT = std::vector<Ptr>; using VecT = std::vector<Ptr>;
static constexpr const char* Name = "Segment"; static constexpr const char* Name = "Segment";
Segment(ID_TYPE partition_id, ID_TYPE num = 0, ID_TYPE id = 0, State status = PENDING, Segment(ID_TYPE partition_id, ID_TYPE num = 0, ID_TYPE id = 0, LSN_TYPE lsn = 0, State status = PENDING,
TS_TYPE created_on = GetMicroSecTimeStamp(), TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); TS_TYPE created_on = GetMicroSecTimeStamp(), TS_TYPE UpdatedOnField = GetMicroSecTimeStamp());
std::string std::string
...@@ -456,6 +484,7 @@ class SegmentCommit : public DBBaseResource<>, ...@@ -456,6 +484,7 @@ class SegmentCommit : public DBBaseResource<>,
public SegmentIdField, public SegmentIdField,
public MappingsField, public MappingsField,
public IdField, public IdField,
public LsnField,
public StatusField, public StatusField,
public CreatedOnField, public CreatedOnField,
public UpdatedOnField { public UpdatedOnField {
...@@ -467,7 +496,7 @@ class SegmentCommit : public DBBaseResource<>, ...@@ -467,7 +496,7 @@ class SegmentCommit : public DBBaseResource<>,
static constexpr const char* Name = "SegmentCommit"; static constexpr const char* Name = "SegmentCommit";
SegmentCommit(ID_TYPE schema_id, ID_TYPE partition_id, ID_TYPE segment_id, const MappingT& mappings = {}, SegmentCommit(ID_TYPE schema_id, ID_TYPE partition_id, ID_TYPE segment_id, const MappingT& mappings = {},
ID_TYPE id = 0, State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(), ID_TYPE id = 0, LSN_TYPE lsn = 0, State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(),
TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); TS_TYPE UpdatedOnField = GetMicroSecTimeStamp());
std::string std::string
...@@ -481,6 +510,7 @@ class SegmentFile : public DBBaseResource<>, ...@@ -481,6 +510,7 @@ class SegmentFile : public DBBaseResource<>,
public SegmentIdField, public SegmentIdField,
public FieldElementIdField, public FieldElementIdField,
public IdField, public IdField,
public LsnField,
public StatusField, public StatusField,
public CreatedOnField, public CreatedOnField,
public UpdatedOnField { public UpdatedOnField {
...@@ -491,7 +521,7 @@ class SegmentFile : public DBBaseResource<>, ...@@ -491,7 +521,7 @@ class SegmentFile : public DBBaseResource<>,
using VecT = std::vector<Ptr>; using VecT = std::vector<Ptr>;
static constexpr const char* Name = "SegmentFile"; static constexpr const char* Name = "SegmentFile";
SegmentFile(ID_TYPE partition_id, ID_TYPE segment_id, ID_TYPE field_element_id, ID_TYPE id = 0, SegmentFile(ID_TYPE partition_id, ID_TYPE segment_id, ID_TYPE field_element_id, ID_TYPE id = 0, LSN_TYPE lsn = 0,
State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(), State status = PENDING, TS_TYPE created_on = GetMicroSecTimeStamp(),
TS_TYPE UpdatedOnField = GetMicroSecTimeStamp()); TS_TYPE UpdatedOnField = GetMicroSecTimeStamp());
}; };
......
core/src/db/snapshot/Snapshot.cpp
浏览文件 @ 295105ff
...@@ -30,6 +30,7 @@ Snapshot::UnRefAll() { ...@@ -30,6 +30,7 @@ Snapshot::UnRefAll() {
Snapshot::Snapshot(ID_TYPE id) { Snapshot::Snapshot(ID_TYPE id) {
auto collection_commit = CollectionCommitsHolder::GetInstance().GetResource(id, false); auto collection_commit = CollectionCommitsHolder::GetInstance().GetResource(id, false);
AddResource<CollectionCommit>(collection_commit); AddResource<CollectionCommit>(collection_commit);
max_lsn_ = collection_commit->GetLsn();
auto& schema_holder = SchemaCommitsHolder::GetInstance(); auto& schema_holder = SchemaCommitsHolder::GetInstance();
auto current_schema = schema_holder.GetResource(collection_commit->GetSchemaId(), false); auto current_schema = schema_holder.GetResource(collection_commit->GetSchemaId(), false);
AddResource<SchemaCommit>(current_schema); AddResource<SchemaCommit>(current_schema);
......
core/src/db/snapshot/Snapshot.h
浏览文件 @ 295105ff
...@@ -76,6 +76,11 @@ class Snapshot : public ReferenceProxy { ...@@ -76,6 +76,11 @@ class Snapshot : public ReferenceProxy {
return GetResources<Partition>().size(); return GetResources<Partition>().size();
} }
const LSN_TYPE&
GetMaxLsn() const {
return max_lsn_;
}
Status Status
GetPartitionId(const std::string& name, ID_TYPE& id) const { GetPartitionId(const std::string& name, ID_TYPE& id) const {
auto it = partition_names_map_.find(name); auto it = partition_names_map_.find(name);
...@@ -252,6 +257,7 @@ class Snapshot : public ReferenceProxy { ...@@ -252,6 +257,7 @@ class Snapshot : public ReferenceProxy {
std::map<ID_TYPE, ID_TYPE> p_pc_map_; std::map<ID_TYPE, ID_TYPE> p_pc_map_;
ID_TYPE latest_schema_commit_id_ = 0; ID_TYPE latest_schema_commit_id_ = 0;
std::map<ID_TYPE, NUM_TYPE> p_max_seg_num_; std::map<ID_TYPE, NUM_TYPE> p_max_seg_num_;
LSN_TYPE max_lsn_;
}; };
using ScopedSnapshotT = ScopedResource<Snapshot>; using ScopedSnapshotT = ScopedResource<Snapshot>;
......
core/src/db/snapshot/Snapshots.cpp
浏览文件 @ 295105ff
...@@ -17,28 +17,29 @@ namespace engine { ...@@ -17,28 +17,29 @@ namespace engine {
namespace snapshot { namespace snapshot {
Status Status
Snapshots::DropCollection(ID_TYPE collection_id) { Snapshots::DropCollection(ID_TYPE collection_id, const LSN_TYPE& lsn) {
ScopedSnapshotT ss; ScopedSnapshotT ss;
auto status = GetSnapshot(ss, collection_id); auto status = GetSnapshot(ss, collection_id);
if (!status.ok()) if (!status.ok())
return status; return status;
return DoDropCollection(ss); return DoDropCollection(ss, lsn);
} }
Status Status
Snapshots::DropCollection(const std::string& name) { Snapshots::DropCollection(const std::string& name, const LSN_TYPE& lsn) {
ScopedSnapshotT ss; ScopedSnapshotT ss;
auto status = GetSnapshot(ss, name); auto status = GetSnapshot(ss, name);
if (!status.ok()) if (!status.ok())
return status; return status;
return DoDropCollection(ss); return DoDropCollection(ss, lsn);
} }
Status Status
Snapshots::DoDropCollection(ScopedSnapshotT& ss) { Snapshots::DoDropCollection(ScopedSnapshotT& ss, const LSN_TYPE& lsn) {
OperationContext context; OperationContext context;
context.lsn = lsn;
context.collection = ss->GetCollection(); context.collection = ss->GetCollection();
auto op = std::make_shared<SoftDeleteCollectionOperation>(context); auto op = std::make_shared<SoftDeleteCollectionOperation>(context, ss);
op->Push(); op->Push();
auto status = op->GetStatus(); auto status = op->GetStatus();
......
core/src/db/snapshot/Snapshots.h
浏览文件 @ 295105ff
...@@ -49,9 +49,9 @@ class Snapshots { ...@@ -49,9 +49,9 @@ class Snapshots {
GetCollectionIds(IDS_TYPE& ids) const; GetCollectionIds(IDS_TYPE& ids) const;
Status Status
DropCollection(const std::string& name); DropCollection(const std::string& name, const LSN_TYPE& lsn);
Status Status
DropCollection(ID_TYPE collection_id); DropCollection(ID_TYPE collection_id, const LSN_TYPE& lsn);
Status Status
Reset(); Reset();
...@@ -63,7 +63,7 @@ class Snapshots { ...@@ -63,7 +63,7 @@ class Snapshots {
Init(); Init();
} }
Status Status
DoDropCollection(ScopedSnapshotT& ss); DoDropCollection(ScopedSnapshotT& ss, const LSN_TYPE& lsn);
void void
Init(); Init();
......
core/src/utils/Error.h
浏览文件 @ 295105ff
...@@ -128,6 +128,7 @@ constexpr ErrorCode SS_DUPLICATED_ERROR = ToSSErrorCode(5); ...@@ -128,6 +128,7 @@ constexpr ErrorCode SS_DUPLICATED_ERROR = ToSSErrorCode(5);
constexpr ErrorCode SS_NOT_ACTIVE_ERROR = ToSSErrorCode(6); constexpr ErrorCode SS_NOT_ACTIVE_ERROR = ToSSErrorCode(6);
constexpr ErrorCode SS_CONSTRAINT_CHECK_ERROR = ToSSErrorCode(7); constexpr ErrorCode SS_CONSTRAINT_CHECK_ERROR = ToSSErrorCode(7);
constexpr ErrorCode SS_INVALID_ARGUMENT_ERROR = ToSSErrorCode(8); constexpr ErrorCode SS_INVALID_ARGUMENT_ERROR = ToSSErrorCode(8);
constexpr ErrorCode SS_OPERATION_PENDING = ToSSErrorCode(9);
namespace server { namespace server {
class ServerException : public std::exception { class ServerException : public std::exception {
......
core/unittest/db/test_snapshot.cpp
浏览文件 @ 295105ff
...@@ -131,8 +131,9 @@ TEST_F(SnapshotTest, ResourceHoldersTest) { ...@@ -131,8 +131,9 @@ TEST_F(SnapshotTest, ResourceHoldersTest) {
} }
milvus::engine::snapshot::ScopedSnapshotT milvus::engine::snapshot::ScopedSnapshotT
CreateCollection(const std::string& collection_name) { CreateCollection(const std::string& collection_name, milvus::engine::snapshot::LSN_TYPE lsn) {
milvus::engine::snapshot::CreateCollectionContext context; milvus::engine::snapshot::CreateCollectionContext context;
context.lsn = lsn;
auto collection_schema = std::make_shared<milvus::engine::snapshot::Collection>(collection_name); auto collection_schema = std::make_shared<milvus::engine::snapshot::Collection>(collection_name);
context.collection = collection_schema; context.collection = collection_schema;
auto vector_field = std::make_shared<milvus::engine::snapshot::Field>("vector", 0); auto vector_field = std::make_shared<milvus::engine::snapshot::Field>("vector", 0);
...@@ -156,7 +157,8 @@ TEST_F(SnapshotTest, CreateCollectionOperationTest) { ...@@ -156,7 +157,8 @@ TEST_F(SnapshotTest, CreateCollectionOperationTest) {
ASSERT_TRUE(!expect_null); ASSERT_TRUE(!expect_null);
std::string collection_name = "test_c1"; std::string collection_name = "test_c1";
auto ss = CreateCollection(collection_name); milvus::engine::snapshot::LSN_TYPE lsn = 1;
auto ss = CreateCollection(collection_name, lsn);
ASSERT_TRUE(ss); ASSERT_TRUE(ss);
milvus::engine::snapshot::ScopedSnapshotT latest_ss; milvus::engine::snapshot::ScopedSnapshotT latest_ss;
...@@ -174,8 +176,9 @@ TEST_F(SnapshotTest, CreateCollectionOperationTest) { ...@@ -174,8 +176,9 @@ TEST_F(SnapshotTest, CreateCollectionOperationTest) {
milvus::engine::snapshot::OperationContext sd_op_ctx; milvus::engine::snapshot::OperationContext sd_op_ctx;
sd_op_ctx.collection = latest_ss->GetCollection(); sd_op_ctx.collection = latest_ss->GetCollection();
sd_op_ctx.lsn = latest_ss->GetMaxLsn() + 1;
ASSERT_TRUE(sd_op_ctx.collection->IsActive()); ASSERT_TRUE(sd_op_ctx.collection->IsActive());
auto sd_op = std::make_shared<milvus::engine::snapshot::SoftDeleteCollectionOperation>(sd_op_ctx); auto sd_op = std::make_shared<milvus::engine::snapshot::SoftDeleteCollectionOperation>(sd_op_ctx, latest_ss);
status = sd_op->Push(); status = sd_op->Push();
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_TRUE(sd_op->GetStatus().ok()); ASSERT_TRUE(sd_op->GetStatus().ok());
...@@ -188,7 +191,8 @@ TEST_F(SnapshotTest, CreateCollectionOperationTest) { ...@@ -188,7 +191,8 @@ TEST_F(SnapshotTest, CreateCollectionOperationTest) {
TEST_F(SnapshotTest, DropCollectionTest) { TEST_F(SnapshotTest, DropCollectionTest) {
milvus::engine::snapshot::Store::GetInstance().DoReset(); milvus::engine::snapshot::Store::GetInstance().DoReset();
std::string collection_name = "test_c1"; std::string collection_name = "test_c1";
auto ss = CreateCollection(collection_name); milvus::engine::snapshot::LSN_TYPE lsn = 1;
auto ss = CreateCollection(collection_name, lsn);
ASSERT_TRUE(ss); ASSERT_TRUE(ss);
milvus::engine::snapshot::ScopedSnapshotT lss; milvus::engine::snapshot::ScopedSnapshotT lss;
auto status = milvus::engine::snapshot::Snapshots::GetInstance().GetSnapshot(lss, collection_name); auto status = milvus::engine::snapshot::Snapshots::GetInstance().GetSnapshot(lss, collection_name);
...@@ -197,31 +201,33 @@ TEST_F(SnapshotTest, DropCollectionTest) { ...@@ -197,31 +201,33 @@ TEST_F(SnapshotTest, DropCollectionTest) {
ASSERT_EQ(ss->GetID(), lss->GetID()); ASSERT_EQ(ss->GetID(), lss->GetID());
auto prev_ss_id = ss->GetID(); auto prev_ss_id = ss->GetID();
auto prev_c_id = ss->GetCollection()->GetID(); auto prev_c_id = ss->GetCollection()->GetID();
status = milvus::engine::snapshot::Snapshots::GetInstance().DropCollection(collection_name); lsn = ss->GetMaxLsn() + 1;
status = milvus::engine::snapshot::Snapshots::GetInstance().DropCollection(collection_name, lsn);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
status = milvus::engine::snapshot::Snapshots::GetInstance().GetSnapshot(lss, collection_name); status = milvus::engine::snapshot::Snapshots::GetInstance().GetSnapshot(lss, collection_name);
ASSERT_TRUE(!status.ok()); ASSERT_TRUE(!status.ok());
auto ss_2 = CreateCollection(collection_name); auto ss_2 = CreateCollection(collection_name, ++lsn);
status = milvus::engine::snapshot::Snapshots::GetInstance().GetSnapshot(lss, collection_name); status = milvus::engine::snapshot::Snapshots::GetInstance().GetSnapshot(lss, collection_name);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(ss_2->GetID(), lss->GetID()); ASSERT_EQ(ss_2->GetID(), lss->GetID());
ASSERT_TRUE(prev_ss_id != ss_2->GetID()); ASSERT_TRUE(prev_ss_id != ss_2->GetID());
ASSERT_TRUE(prev_c_id != ss_2->GetCollection()->GetID()); ASSERT_TRUE(prev_c_id != ss_2->GetCollection()->GetID());
status = milvus::engine::snapshot::Snapshots::GetInstance().DropCollection(collection_name); status = milvus::engine::snapshot::Snapshots::GetInstance().DropCollection(collection_name, ++lsn);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
status = milvus::engine::snapshot::Snapshots::GetInstance().DropCollection(collection_name); status = milvus::engine::snapshot::Snapshots::GetInstance().DropCollection(collection_name, ++lsn);
ASSERT_TRUE(!status.ok()); ASSERT_TRUE(!status.ok());
} }
TEST_F(SnapshotTest, ConCurrentCollectionOperation) { TEST_F(SnapshotTest, ConCurrentCollectionOperation) {
milvus::engine::snapshot::Store::GetInstance().DoReset(); milvus::engine::snapshot::Store::GetInstance().DoReset();
std::string collection_name("c1"); std::string collection_name("c1");
milvus::engine::snapshot::LSN_TYPE lsn = 1;
milvus::engine::snapshot::ID_TYPE stale_ss_id; milvus::engine::snapshot::ID_TYPE stale_ss_id;
auto worker1 = [&]() { auto worker1 = [&]() {
milvus::Status status; milvus::Status status;
auto ss = CreateCollection(collection_name); auto ss = CreateCollection(collection_name, ++lsn);
ASSERT_TRUE(ss); ASSERT_TRUE(ss);
ASSERT_EQ(ss->GetName(), collection_name); ASSERT_EQ(ss->GetName(), collection_name);
stale_ss_id = ss->GetID(); stale_ss_id = ss->GetID();
...@@ -239,7 +245,7 @@ TEST_F(SnapshotTest, ConCurrentCollectionOperation) { ...@@ -239,7 +245,7 @@ TEST_F(SnapshotTest, ConCurrentCollectionOperation) {
}; };
auto worker2 = [&] { auto worker2 = [&] {
std::this_thread::sleep_for(std::chrono::milliseconds(50)); std::this_thread::sleep_for(std::chrono::milliseconds(50));
auto status = milvus::engine::snapshot::Snapshots::GetInstance().DropCollection(collection_name); auto status = milvus::engine::snapshot::Snapshots::GetInstance().DropCollection(collection_name, ++lsn);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
milvus::engine::snapshot::ScopedSnapshotT a_ss; milvus::engine::snapshot::ScopedSnapshotT a_ss;
status = milvus::engine::snapshot::Snapshots::GetInstance().GetSnapshot(a_ss, collection_name); status = milvus::engine::snapshot::Snapshots::GetInstance().GetSnapshot(a_ss, collection_name);
...@@ -247,10 +253,10 @@ TEST_F(SnapshotTest, ConCurrentCollectionOperation) { ...@@ -247,10 +253,10 @@ TEST_F(SnapshotTest, ConCurrentCollectionOperation) {
}; };
auto worker3 = [&] { auto worker3 = [&] {
std::this_thread::sleep_for(std::chrono::milliseconds(20)); std::this_thread::sleep_for(std::chrono::milliseconds(20));
auto ss = CreateCollection(collection_name); auto ss = CreateCollection(collection_name, ++lsn);
ASSERT_TRUE(!ss); ASSERT_TRUE(!ss);
std::this_thread::sleep_for(std::chrono::milliseconds(80)); std::this_thread::sleep_for(std::chrono::milliseconds(80));
ss = CreateCollection(collection_name); ss = CreateCollection(collection_name, ++lsn);
ASSERT_TRUE(ss); ASSERT_TRUE(ss);
ASSERT_EQ(ss->GetName(), collection_name); ASSERT_EQ(ss->GetName(), collection_name);
}; };
...@@ -268,12 +274,14 @@ TEST_F(SnapshotTest, ConCurrentCollectionOperation) { ...@@ -268,12 +274,14 @@ TEST_F(SnapshotTest, ConCurrentCollectionOperation) {
TEST_F(SnapshotTest, PartitionTest) { TEST_F(SnapshotTest, PartitionTest) {
milvus::engine::snapshot::Store::GetInstance().DoReset(); milvus::engine::snapshot::Store::GetInstance().DoReset();
std::string collection_name("c1"); std::string collection_name("c1");
auto ss = CreateCollection(collection_name); milvus::engine::snapshot::LSN_TYPE lsn = 1;
auto ss = CreateCollection(collection_name, ++lsn);
ASSERT_TRUE(ss); ASSERT_TRUE(ss);
ASSERT_EQ(ss->GetName(), collection_name); ASSERT_EQ(ss->GetName(), collection_name);
ASSERT_EQ(ss->NumberOfPartitions(), 1); ASSERT_EQ(ss->NumberOfPartitions(), 1);
milvus::engine::snapshot::OperationContext context; milvus::engine::snapshot::OperationContext context;
context.lsn = ++lsn;
auto op = std::make_shared<milvus::engine::snapshot::CreatePartitionOperation>(context, ss); auto op = std::make_shared<milvus::engine::snapshot::CreatePartitionOperation>(context, ss);
std::string partition_name("p1"); std::string partition_name("p1");
...@@ -297,6 +305,7 @@ TEST_F(SnapshotTest, PartitionTest) { ...@@ -297,6 +305,7 @@ TEST_F(SnapshotTest, PartitionTest) {
ASSERT_TRUE(curr_ss->GetID() > ss->GetID()); ASSERT_TRUE(curr_ss->GetID() > ss->GetID());
ASSERT_EQ(curr_ss->NumberOfPartitions(), 2); ASSERT_EQ(curr_ss->NumberOfPartitions(), 2);
p_ctx.lsn = ++lsn;
auto drop_op = std::make_shared<milvus::engine::snapshot::DropPartitionOperation>(p_ctx, curr_ss); auto drop_op = std::make_shared<milvus::engine::snapshot::DropPartitionOperation>(p_ctx, curr_ss);
status = drop_op->Push(); status = drop_op->Push();
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
...@@ -309,15 +318,45 @@ TEST_F(SnapshotTest, PartitionTest) { ...@@ -309,15 +318,45 @@ TEST_F(SnapshotTest, PartitionTest) {
ASSERT_TRUE(latest_ss->GetID() > curr_ss->GetID()); ASSERT_TRUE(latest_ss->GetID() > curr_ss->GetID());
ASSERT_EQ(latest_ss->NumberOfPartitions(), 1); ASSERT_EQ(latest_ss->NumberOfPartitions(), 1);
p_ctx.lsn = ++lsn;
drop_op = std::make_shared<milvus::engine::snapshot::DropPartitionOperation>(p_ctx, latest_ss); drop_op = std::make_shared<milvus::engine::snapshot::DropPartitionOperation>(p_ctx, latest_ss);
status = drop_op->Push(); status = drop_op->Push();
ASSERT_TRUE(!status.ok()); ASSERT_TRUE(!status.ok());
std::cout << status.ToString() << std::endl; std::cout << status.ToString() << std::endl;
} }
TEST_F(SnapshotTest, PartitionTest2) {
milvus::engine::snapshot::Store::GetInstance().DoReset();
std::string collection_name("c1");
milvus::engine::snapshot::LSN_TYPE lsn = 1;
milvus::Status status;
auto ss = CreateCollection(collection_name, ++lsn);
ASSERT_TRUE(ss);
ASSERT_EQ(lsn, ss->GetMaxLsn());
milvus::engine::snapshot::OperationContext context;
context.lsn = lsn;
auto cp_op = std::make_shared<milvus::engine::snapshot::CreatePartitionOperation>(context, ss);
std::string partition_name("p1");
milvus::engine::snapshot::PartitionContext p_ctx;
p_ctx.name = partition_name;
milvus::engine::snapshot::PartitionPtr partition;
status = cp_op->CommitNewPartition(p_ctx, partition);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(partition);
ASSERT_EQ(partition->GetName(), partition_name);
ASSERT_TRUE(!partition->IsActive());
ASSERT_TRUE(partition->HasAssigned());
status = cp_op->Push();
ASSERT_TRUE(!status.ok());
}
TEST_F(SnapshotTest, OperationTest) { TEST_F(SnapshotTest, OperationTest) {
milvus::Status status; milvus::Status status;
std::string to_string; std::string to_string;
milvus::engine::snapshot::LSN_TYPE lsn;
milvus::engine::snapshot::SegmentFileContext sf_context; milvus::engine::snapshot::SegmentFileContext sf_context;
sf_context.field_name = "f_1_1"; sf_context.field_name = "f_1_1";
sf_context.field_element_name = "fe_1_1"; sf_context.field_element_name = "fe_1_1";
...@@ -327,6 +366,7 @@ TEST_F(SnapshotTest, OperationTest) { ...@@ -327,6 +366,7 @@ TEST_F(SnapshotTest, OperationTest) {
milvus::engine::snapshot::ScopedSnapshotT ss; milvus::engine::snapshot::ScopedSnapshotT ss;
status = milvus::engine::snapshot::Snapshots::GetInstance().GetSnapshot(ss, 1); status = milvus::engine::snapshot::Snapshots::GetInstance().GetSnapshot(ss, 1);
auto ss_id = ss->GetID(); auto ss_id = ss->GetID();
lsn = ss->GetMaxLsn() + 1;
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
// Check snapshot // Check snapshot
...@@ -347,6 +387,7 @@ TEST_F(SnapshotTest, OperationTest) { ...@@ -347,6 +387,7 @@ TEST_F(SnapshotTest, OperationTest) {
// Check build operation correctness // Check build operation correctness
{ {
milvus::engine::snapshot::OperationContext context; milvus::engine::snapshot::OperationContext context;
context.lsn = ++lsn;
auto build_op = std::make_shared<milvus::engine::snapshot::BuildOperation>(context, ss); auto build_op = std::make_shared<milvus::engine::snapshot::BuildOperation>(context, ss);
milvus::engine::snapshot::SegmentFilePtr seg_file; milvus::engine::snapshot::SegmentFilePtr seg_file;
status = build_op->CommitNewSegmentFile(sf_context, seg_file); status = build_op->CommitNewSegmentFile(sf_context, seg_file);
...@@ -383,6 +424,7 @@ TEST_F(SnapshotTest, OperationTest) { ...@@ -383,6 +424,7 @@ TEST_F(SnapshotTest, OperationTest) {
milvus::engine::snapshot::ID_TYPE partition_id; milvus::engine::snapshot::ID_TYPE partition_id;
{ {
milvus::engine::snapshot::OperationContext context; milvus::engine::snapshot::OperationContext context;
context.lsn = ++lsn;
context.prev_partition = ss->GetResource<milvus::engine::snapshot::Partition>(1); context.prev_partition = ss->GetResource<milvus::engine::snapshot::Partition>(1);
auto op = std::make_shared<milvus::engine::snapshot::NewSegmentOperation>(context, ss); auto op = std::make_shared<milvus::engine::snapshot::NewSegmentOperation>(context, ss);
milvus::engine::snapshot::SegmentPtr new_seg; milvus::engine::snapshot::SegmentPtr new_seg;
...@@ -420,6 +462,7 @@ TEST_F(SnapshotTest, OperationTest) { ...@@ -420,6 +462,7 @@ TEST_F(SnapshotTest, OperationTest) {
auto expect_null = ss->GetPartitionCommitByPartitionId(11111111); auto expect_null = ss->GetPartitionCommitByPartitionId(11111111);
ASSERT_TRUE(!expect_null); ASSERT_TRUE(!expect_null);
ASSERT_NE(prev_partition_commit->ToString(), ""); ASSERT_NE(prev_partition_commit->ToString(), "");
merge_ctx.lsn = ++lsn;
auto op = std::make_shared<milvus::engine::snapshot::MergeOperation>(merge_ctx, ss); auto op = std::make_shared<milvus::engine::snapshot::MergeOperation>(merge_ctx, ss);
milvus::engine::snapshot::SegmentPtr new_seg; milvus::engine::snapshot::SegmentPtr new_seg;
status = op->CommitNewSegment(new_seg); status = op->CommitNewSegment(new_seg);
...@@ -455,6 +498,7 @@ TEST_F(SnapshotTest, OperationTest) { ...@@ -455,6 +498,7 @@ TEST_F(SnapshotTest, OperationTest) {
// 4. Commit new seg file of build operation -> Stale Segment Found Here! // 4. Commit new seg file of build operation -> Stale Segment Found Here!
{ {
milvus::engine::snapshot::OperationContext context; milvus::engine::snapshot::OperationContext context;
context.lsn = ++lsn;
auto build_op = std::make_shared<milvus::engine::snapshot::BuildOperation>(context, new_ss); auto build_op = std::make_shared<milvus::engine::snapshot::BuildOperation>(context, new_ss);
milvus::engine::snapshot::SegmentFilePtr seg_file; milvus::engine::snapshot::SegmentFilePtr seg_file;
auto new_sf_context = sf_context; auto new_sf_context = sf_context;
...@@ -469,6 +513,7 @@ TEST_F(SnapshotTest, OperationTest) { ...@@ -469,6 +513,7 @@ TEST_F(SnapshotTest, OperationTest) {
// 4. Commit build operation -> Stale Segment Found Here! // 4. Commit build operation -> Stale Segment Found Here!
{ {
milvus::engine::snapshot::OperationContext context; milvus::engine::snapshot::OperationContext context;
context.lsn = ++lsn;
auto build_op = std::make_shared<milvus::engine::snapshot::BuildOperation>(context, ss); auto build_op = std::make_shared<milvus::engine::snapshot::BuildOperation>(context, ss);
milvus::engine::snapshot::SegmentFilePtr seg_file; milvus::engine::snapshot::SegmentFilePtr seg_file;
auto new_sf_context = sf_context; auto new_sf_context = sf_context;
...@@ -477,7 +522,8 @@ TEST_F(SnapshotTest, OperationTest) { ...@@ -477,7 +522,8 @@ TEST_F(SnapshotTest, OperationTest) {
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
std::cout << build_op->ToString() << std::endl; std::cout << build_op->ToString() << std::endl;
auto status = milvus::engine::snapshot::Snapshots::GetInstance().DropCollection(ss->GetName()); auto status = milvus::engine::snapshot::Snapshots::GetInstance().DropCollection(ss->GetName(),
++lsn);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
status = build_op->Push(); status = build_op->Push();
ASSERT_TRUE(!status.ok()); ASSERT_TRUE(!status.ok());
......
core/unittest/db/utils.cpp
浏览文件 @ 295105ff
...@@ -378,6 +378,8 @@ SnapshotTest::SetUp() { ...@@ -378,6 +378,8 @@ SnapshotTest::SetUp() {
void void
SnapshotTest::TearDown() { SnapshotTest::TearDown() {
// TODO: Temp to delay some time. OperationExecutor should wait all resources be destructed before stop
std::this_thread::sleep_for(std::chrono::milliseconds(20));
milvus::engine::snapshot::OperationExecutor::GetInstance().Stop(); milvus::engine::snapshot::OperationExecutor::GetInstance().Stop();
BaseTest::TearDown(); BaseTest::TearDown();
} }
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或