add replay_fd_bug.sh

rdsn @ a6f5d11a

-Subproject commit 38d7c02c89e22865e16775ed7e0f8e2f2f4732e5
+Subproject commit a6f5d11a9c914acdb61b5c86732eddd4e9a83481

replay_fd_bug.sh

+#!/bin/bash
+
+./run.sh clear_onebox
+echo
+
+./run.sh start_onebox -r 6 -w -p 16
+echo
+
+primary=`echo cluster_info | ./run.sh shell 2>&1 | grep primary_meta_server | awk '{print $3}'`
+pid=`echo "app temp -d" | ./run.sh shell 2>&1 | grep " 10.239.35.112:34801 " | head -n 1 |awk '{print $1}'`
+
+echo "set meta.fd.ignore = 10.239.35.112:34801"
+echo "remote_command -l $primary meta.fd.ignore 10.239.35.112:34801" | ./run.sh shell &>/dev/null
+echo "sleep 20 (to wait replica1 become unalive)"
+sleep 20
+echo "nodes -d" | ./run.sh shell 2>&1 | grep "10.239.35.112:34801"
+echo
+
+echo "set meta.fd.ignore = none"
+echo "remote_command -l $primary meta.fd.ignore none" | ./run.sh shell &>/dev/null
+echo "sleep 5 (to wait replica1 become alive)"
+sleep 5
+echo "nodes -d" | ./run.sh shell 2>&1 | grep "10.239.35.112:34801"
+echo
+
+echo "set meta.lb.assign_delay_ms = 1000"
+echo "remote_command -l $primary meta.lb.assign_delay_ms 1000" | ./run.sh shell &>/dev/null
+echo "sleep 5 (to wait replica1 coredump)"
+sleep 5
+
+echo "find core in onebox/replica1:"
+echo "------------------------------"
+grep "^F" onebox/replica1/data/log/log.1.txt
+

src/server/config-server.ini

@@ -44,7 +44,7 @@ cli_remote = true
 ;aio_factory_name = dsn::tools::native_aio_provider
 start_nfs = true
 
-logging_start_level = LOG_LEVEL_DEBUG
+logging_start_level = LOG_LEVEL_INFORMATION
 logging_factory_name = dsn::tools::simple_logger
 ;logging_factory_name = dsn::tools::screen_logger
 ;logging_factory_name = dsn::tools::hpc_logger
@@ -150,7 +150,7 @@ allow_inline = false
 fast_execution_in_network_thread = false
 rpc_call_header_format = NET_HDR_DSN
 rpc_call_channel = RPC_CHANNEL_TCP
-rpc_timeout_milliseconds = 5000
+rpc_timeout_milliseconds = 1000
 
 disk_write_fail_ratio = 0.0
 disk_read_fail_ratio = 0.0
@@ -169,7 +169,7 @@ replica_assign_delay_ms_for_dropouts = 300000
 node_live_percentage_threshold_for_update = 50
 min_live_node_count_for_unfreeze = 3
 hold_seconds_for_dropped_app = 604800
-meta_function_level_on_start = lively
+meta_function_level_on_start = steady
 recover_from_replica_server = false
 max_replicas_in_group = 4
 
@@ -204,7 +204,7 @@ staleness_for_commit = 20
 max_mutation_count_in_prepare_list = 110
 mutation_2pc_min_replica_count = 2
 
-group_check_disabled = false
+group_check_disabled = true
 group_check_interval_ms = 100000
 
 checkpoint_disabled = false
@@ -213,7 +213,7 @@ checkpoint_min_decree_gap = 10000
 checkpoint_max_interval_hours = 1
 
 gc_disabled = false
-gc_interval_ms = 30000
+gc_interval_ms = 2000
 gc_memory_replica_interval_ms = 300000
 gc_disk_error_replica_interval_seconds = 172800000
 gc_disk_garbage_replica_interval_seconds = 86400
@@ -241,7 +241,7 @@ log_shared_force_flush = false
 log_shared_pending_size_throttling_threshold_kb = 0
 log_shared_pending_size_throttling_delay_ms = 0
 
-config_sync_disabled = false
+config_sync_disabled = true
 config_sync_interval_ms = 30000
 
 lb_interval_ms = 3000