ITPub博客

首页 > 数据库 > 数据库开发技术 > RocksDB重要参数索引

RocksDB重要参数索引

原创 数据库开发技术 作者:jesselyu 时间:2017-04-05 22:30:05 0 删除 编辑

Block配置

#block级别的cache=size,类似于innodb=buffer=pool

rocksdb_block_cache_size=96636764160

rocksdb_no_block_cache=OFF

rocksdb_block_restart_interval=16

#块大小,类似于innodbpage=size

rocksdb_block_size=16384

rocksdb_block_size_deviation=10

导数优化

rocksdb_bulk_load=OFF

#每次loadbatch=size

rocksdb_bulk_load_size=1000

#对批量load进行拆分

#如果load失败,会有部分数据导入成功,在下次导入前需要truncate

rocksdb_commit_in_the_middle=OFF

 

数据目录配置

#数据目录

rocksdb_datadir=./.rocksdb

#是否开起共享的write_buffer_size,默认关闭

#因为每个column=family级别已经配置了write_buffer_size

rocksdb_db_write_buffer_size=0

rocksdb_debug_optimizer_no_zero_cardinality=ON

 

Column Family相关配置

rocksdb_default_cf_options=write_buffer_size=128m;target_file_size_base=32m;max_bytes_for_level_base=512m;level0_file_num_compaction_trigger=4;level0_slowdown_writes_trigger=10;level0_stop_writes_trigger=15;max_write_buffer_number=4;compression_per_level=kNoCompression:kNoCompression:kNoCompression:kLZ4Compression:kLZ4Compression:kLZ4Compression;bottommost_compression=kZlibCompression;compression_opts=-14:1:0;block_based_table_factory={cache_index_and_filter_blocks=1;filter_policy=bloomfilter:10:false;whole_key_filtering=1};level_compaction_dynamic_level_bytes=true;optimize_filters_for_hits=true

 

1.CF参数解析

write_buffer_size=64M,也是memtable的大小,因为flush都是基于memtable

max_write_buffer_number=2,一个CF中允许的最大writer buffer

min_write_buffer_number_to_merge=1,如果是“1”表示在flush时,不作任何合并操作

max_write_buffer_number_to_maintain=(max_write_buffer_number|0),“0”表示writer_buffer flush之后立即释放内存,如果是这样,当事务进行写

                                                                          冲突检查时,可能需要读取SST Table,降低性能。

Compression=(kSnappyCompress,kNoCompression),内存block级别的compress

 // Typical speeds of kSnappyCompression on an Intel(R) Core(TM)2 2.4GHz:

  //    ~200-500MB/s compression

  //    ~400-800MB/s decompression

 

level0_file_num_compaction_trigger=4level0达到4sst文件时开始compaction

level0_slowdown_writes_trigger=10level0超出10sst文件,将降低写入速度到(delayed_write_rate=2M/s),默认为”-1”禁用

level0_stop_writes_trigger=15level0超出15个文件后将禁止写入,默认为”-1”禁用

target_file_size_base=64Mlevel-1sst文件大小

target_file_size_multiplier=10,每级sst文件大小的放大系数,每级sst文件大小为:target_file_size_base* target_file_size_multiplier^(level-1)

max_bytes_for_level_base =256M,为level-1sst文件最大size,每级算法同上

max_bytes_for_level_multiplier =10,作用同上

disable_auto_compactions=true,禁止自动compaction,但是CF级别的手动compaction还是可以的

soft_pending_compaction_bytes_limit=64G,如果超出64G需要compaction,那写入率会降低到delayed_write_rate=2M/s

hard_pending_compaction_bytes_limit=256G,如果超出256G,则禁止写入

 

2.Bloom Filter参数解析

block_based_table_factory={cache_index_and_filter_blocks=1;filter_policy=bloomfilter:10:false;whole_key_filtering=1}

cache_index_and_filter_blocks:缓存 filter 

filter_policy:起用bloom filterhash函数为10

whole_key_filtering:起用全键模式filter

3.SST压缩算法参数解析

compression_per_level=kNoCompression:kNoCompression:kNoCompression:kZlibCompression:kZlibCompression:kZlibCompression:kZlibCompression

compression_opts=- 14:6:0

这里支持的压缩算法有:Snappy(主要用于level-0level-1)Zlib(default)LZ4Bzip

 

 

 

 

#增加覆盖默认配置

rocksdb_override_cf_options

 

rocksdb_create_missing_column_families=OFF

rocksdb_write_ignore_missing_column_families=OFF

 

log file相关配置

rocksdb_log_file_time_to_roll=0

rocksdb_max_log_file_size=0

#保留的日志数

rocksdb_keep_log_file_num=1000

#日志log=level

rocksdb_info_log_level=error_level

 

Compaction相关配置

#compaction后台线程,在低优先级队列中

rocksdb_base_background_compactions=1

#compaction最大后台线程,在低优先级队列中

rocksdb_max_background_compactions=8

#是否支持compaction并行度,为“1”时,不支持

rocksdb_max_subcompactions=1

rocksdb_compact_cf

rocksdb_compaction_readahead_size=0

#顺序删除1999999key时,触发compaction

rocksdb_compaction_sequential_deletes=199999

rocksdb_compaction_sequential_deletes_count_sd=ON

rocksdb_compaction_sequential_deletes_file_size=0

rocksdb_compaction_sequential_deletes_window=200000

rocksdb_new_table_reader_for_compaction_inputs=OFF

MemTable配置

#最大后台flushes线程数,在高优先级队列中

rocksdb_max_background_flushes=4

rocksdb_flush_memtable_on_analyze=ON

rocksdb_force_flush_memtable_now=OFF

rocksdb_allow_concurrent_memtable_write=OFF

MANIFIST文件配置

rocksdb_max_manifest_file_size=18446744073709551615

rocksdb_manifest_preallocation_size=4194304

 

锁相关配置

#最大行锁,如果超出会导致加锁失败

rocksdb_max_row_locks=1073741824

rocksdb_lock_scanned_rows=OFF

#行锁等待超时

rocksdb_lock_wait_timeout=2

 

 

文件句柄配置

#不限制文件句柄,被opentable不会关闭

rocksdb_max_open_files=-1

 

写入速度控制

#控制最大写出速度为100M/s

rocksdb_rate_limiter_bytes_per_sec=104857600

 

 

Bloom Filter配置

#起用read时的bloom filter

rocksdb_skip_bloom_filter_on_read=OFF

 

 

字符校验

#索引大小写检查

rocksdb_strict_collation_check=ON

rocksdb_strict_collation_exceptions

刷盘方式

#关闭fsyncsync 文件metadata,但是性能损耗会大些

rocksdb_use_fsync=OFF

#关闭后台sync动作

rocksdb_background_sync=OFF

#4k sector对齐

rocksdb_bytes_per_sync=4194304

#起用datasync,文件meta信息不会被刷新,可能会导致corrupted block

rocksdb_disabledatasync=OFF

Check Sum配置

rocksdb_validate_tables=1

rocksdb_verify_checksums=OFF

rocksdb_checksums_pct=100

rocksdb_store_checksums=OFF

#rocksdb会检查put,write等操作,如果失败则将库变成“read-only“状态

rocksdb_paranoid_checks=ON

WAL日志相关参数

rocksdb_max_total_wal_size=4294967296

rocksdb_wal_bytes_per_sync=4194304

rocksdb_wal_dir

rocksdb_wal_recovery_mode=2

rocksdb_wal_size_limit_mb=0

rocksdb_wal_ttl_seconds=0

rocksdb_whole_key_filtering=ON

rocksdb_write_disable_wal=OFF

 

rocksdb_write_sync=OFF

 

 

Table Cache

#table=cache分片

rocksdb_table_cache_numshardbits=6

OS行为相关参数

rocksdb_allow_mmap_reads=OFF

rocksdb_allow_mmap_writes=OFF

#是否允许disk=I/O进行system=cache

rocksdb_allow_os_buffer=ON

 

 

检查点Checkpoint

rocksdb_create_checkpoint

 

 

线程配置

rocksdb_enable_thread_tracking=OFF

rocksdb_enable_write_thread_adaptive_yield=OFF

#停止所有rocksdb后台线程

rocksdb_pause_background_work=OFF

其它

rocksdb_perf_context_level=0

rocksdb_pin_l0_filter_and_index_blocks_in_cache=ON

rocksdb_records_in_range=0

rocksdb_rpl_lookup_rows=OFF

rocksdb_seconds_between_stat_computes=3600

rocksdb_signal_drop_index_thread=OFF

rocksdb_table_stats_sampling_pct=10

rocksdb_unsafe_for_binlog=OFF

rocksdb_use_adaptive_mutex=OFF

 

 

rocksdb_create_if_missing=ON

rocksdb_access_hint_on_compaction_start=1

rocksdb_advise_random_on_open=ON

 

 

rocksdb_force_index_records_in_range=0

rocksdb_hash_index_allow_collision=ON

 

rocksdb_is_fd_close_on_exec=ON

rocksdb_skip_fill_cache=OFF

rocksdb_skip_unique_check=OFF

rocksdb_stats_dump_period_sec=600



核心参数配置模板

#dir

rocksdb_datadir=/u01/my3306/data/rocksdb

rocksdb_wal_dir=/u01/my3306/data/rocksdb


#small 35G,middle 50G, large 70G, xlarge 90G

rocksdb_block_cache_size=90G

rocksdb_max_total_wal_size=4294967296

rocksdb_block_size=16K

rocksdb_write_sync=OFF

rocksdb_table_cache_numshardbits=6

 

#limit write rate 100M/s

rocksdb_rate_limiter_bytes_per_sec=104857600


 

#memtable

rocksdb_max_background_flushes=4

rocksdb_flush_memtable_on_analyze=ON

rocksdb_force_flush_memtable_now=OFF

rocksdb_allow_concurrent_memtable_write=OFF

 

#compaction thread

rocksdb_base_background_compactions=2

rocksdb_max_background_compactions=8

rocksdb_max_subcompactions=1

rocksdb_skip_bloom_filter_on_read=OFF

 

#sector adjust,smoonthly the IO write

rocksdb_bytes_per_sync=4194304

rocksdb_wal_bytes_per_sync=4194304

 

#column family

rocksdb_default_cf_options=write_buffer_size=128m;target_file_size_base=32m;max_bytes_for_level_base=512m;level0_file_num_compaction_trigger=4;level0_slowdown_writes_trigger=10;level0_stop_writes_trigger=15;max_write_buffer_number=4;compression_per_level=kNoCompression:kNoCompression:kNoCompression:kLZ4Compression:kLZ4Compression:kLZ4Compression;bottommost_compression=kZlibCompression;compression_opts=-14:1:0;block_based_table_factory={cache_index_and_filter_blocks=1;filter_policy=bloomfilter:10:false;whole_key_filtering=1};level_compaction_dynamic_level_bytes=true;optimize_filters_for_hits=true

 

 

#bulk load

rocksdb_bulk_load=off

rocksdb_bulk_load_size=10000

rocksdb_commit_in_the_middle=off

#other

rocksdb_strict_collation_check=on

rocksdb_lock_wait_timeout=5

rocksdb_max_open_files=-1

 



来自 “ ITPUB博客 ” ,链接:http://blog.itpub.net/30088583/viewspace-2136672/,如需转载,请注明出处,否则将追究法律责任。

下一篇: RocksDB引擎加载
请登录后发表评论 登录
全部评论
资深Oracle RAC及Exadata部署实施运维专家,曾从事PostgreSQL内核开发与性能优化。丰富的大规模数据库管理架构设计经验。目前在阿里巴巴从事数据库架构工作以及新技术引进,对分布式存储,云计算及并行编程有一定研究。技术讨论请加微信:ljs521688

注册时间:2015-01-01

  • 博文量
    41
  • 访问量
    557965