420 lines
15 KiB
Go
420 lines
15 KiB
Go
/*
|
|
* Copyright 2017 Dgraph Labs, Inc. and Contributors
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package badger
|
|
|
|
import (
|
|
"github.com/dgraph-io/badger/options"
|
|
)
|
|
|
|
// Note: If you add a new option X make sure you also add a WithX method on Options.
|
|
|
|
// Options are params for creating DB object.
|
|
//
|
|
// This package provides DefaultOptions which contains options that should
|
|
// work for most applications. Consider using that as a starting point before
|
|
// customizing it for your own needs.
|
|
//
|
|
// Each option X is documented on the WithX method.
|
|
type Options struct {
|
|
// Required options.
|
|
|
|
Dir string
|
|
ValueDir string
|
|
|
|
// Usually modified options.
|
|
|
|
SyncWrites bool
|
|
TableLoadingMode options.FileLoadingMode
|
|
ValueLogLoadingMode options.FileLoadingMode
|
|
NumVersionsToKeep int
|
|
ReadOnly bool
|
|
Truncate bool
|
|
Logger Logger
|
|
EventLogging bool
|
|
|
|
// Fine tuning options.
|
|
|
|
MaxTableSize int64
|
|
LevelSizeMultiplier int
|
|
MaxLevels int
|
|
ValueThreshold int
|
|
NumMemtables int
|
|
|
|
NumLevelZeroTables int
|
|
NumLevelZeroTablesStall int
|
|
|
|
LevelOneSize int64
|
|
ValueLogFileSize int64
|
|
ValueLogMaxEntries uint32
|
|
|
|
NumCompactors int
|
|
CompactL0OnClose bool
|
|
LogRotatesToFlush int32
|
|
// When set, checksum will be validated for each entry read from the value log file.
|
|
VerifyValueChecksum bool
|
|
|
|
// BypassLockGaurd will bypass the lock guard on badger. Bypassing lock
|
|
// guard can cause data corruption if multiple badger instances are using
|
|
// the same directory. Use this options with caution.
|
|
BypassLockGuard bool
|
|
|
|
// Transaction start and commit timestamps are managed by end-user.
|
|
// This is only useful for databases built on top of Badger (like Dgraph).
|
|
// Not recommended for most users.
|
|
managedTxns bool
|
|
|
|
// 4. Flags for testing purposes
|
|
// ------------------------------
|
|
maxBatchCount int64 // max entries in batch
|
|
maxBatchSize int64 // max batch size in bytes
|
|
|
|
}
|
|
|
|
// DefaultOptions sets a list of recommended options for good performance.
|
|
// Feel free to modify these to suit your needs with the WithX methods.
|
|
func DefaultOptions(path string) Options {
|
|
return Options{
|
|
Dir: path,
|
|
ValueDir: path,
|
|
LevelOneSize: 256 << 20,
|
|
LevelSizeMultiplier: 10,
|
|
TableLoadingMode: options.MemoryMap,
|
|
ValueLogLoadingMode: options.MemoryMap,
|
|
// table.MemoryMap to mmap() the tables.
|
|
// table.Nothing to not preload the tables.
|
|
MaxLevels: 7,
|
|
MaxTableSize: 64 << 20,
|
|
NumCompactors: 2, // Compactions can be expensive. Only run 2.
|
|
NumLevelZeroTables: 5,
|
|
NumLevelZeroTablesStall: 10,
|
|
NumMemtables: 5,
|
|
SyncWrites: true,
|
|
NumVersionsToKeep: 1,
|
|
CompactL0OnClose: true,
|
|
VerifyValueChecksum: false,
|
|
// Nothing to read/write value log using standard File I/O
|
|
// MemoryMap to mmap() the value log files
|
|
// (2^30 - 1)*2 when mmapping < 2^31 - 1, max int32.
|
|
// -1 so 2*ValueLogFileSize won't overflow on 32-bit systems.
|
|
ValueLogFileSize: 1<<30 - 1,
|
|
|
|
ValueLogMaxEntries: 1000000,
|
|
ValueThreshold: 32,
|
|
Truncate: false,
|
|
Logger: defaultLogger,
|
|
EventLogging: true,
|
|
LogRotatesToFlush: 2,
|
|
}
|
|
}
|
|
|
|
// LSMOnlyOptions follows from DefaultOptions, but sets a higher ValueThreshold
|
|
// so values would be colocated with the LSM tree, with value log largely acting
|
|
// as a write-ahead log only. These options would reduce the disk usage of value
|
|
// log, and make Badger act more like a typical LSM tree.
|
|
func LSMOnlyOptions(path string) Options {
|
|
// Max value length which fits in uint16.
|
|
// Let's not set any other options, because they can cause issues with the
|
|
// size of key-value a user can pass to Badger. For e.g., if we set
|
|
// ValueLogFileSize to 64MB, a user can't pass a value more than that.
|
|
// Setting it to ValueLogMaxEntries to 1000, can generate too many files.
|
|
// These options are better configured on a usage basis, than broadly here.
|
|
// The ValueThreshold is the most important setting a user needs to do to
|
|
// achieve a heavier usage of LSM tree.
|
|
// NOTE: If a user does not want to set 64KB as the ValueThreshold because
|
|
// of performance reasons, 1KB would be a good option too, allowing
|
|
// values smaller than 1KB to be colocated with the keys in the LSM tree.
|
|
return DefaultOptions(path).WithValueThreshold(65500)
|
|
}
|
|
|
|
// WithDir returns a new Options value with Dir set to the given value.
|
|
//
|
|
// Dir is the path of the directory where key data will be stored in.
|
|
// If it doesn't exist, Badger will try to create it for you.
|
|
// This is set automatically to be the path given to `DefaultOptions`.
|
|
func (opt Options) WithDir(val string) Options {
|
|
opt.Dir = val
|
|
return opt
|
|
}
|
|
|
|
// WithValueDir returns a new Options value with ValueDir set to the given value.
|
|
//
|
|
// ValueDir is the path of the directory where value data will be stored in.
|
|
// If it doesn't exist, Badger will try to create it for you.
|
|
// This is set automatically to be the path given to `DefaultOptions`.
|
|
func (opt Options) WithValueDir(val string) Options {
|
|
opt.ValueDir = val
|
|
return opt
|
|
}
|
|
|
|
// WithSyncWrites returns a new Options value with SyncWrites set to the given value.
|
|
//
|
|
// When SyncWrites is true all writes are synced to disk. Setting this to false would achieve better
|
|
// performance, but may cause data loss in case of crash.
|
|
//
|
|
// The default value of SyncWrites is true.
|
|
func (opt Options) WithSyncWrites(val bool) Options {
|
|
opt.SyncWrites = val
|
|
return opt
|
|
}
|
|
|
|
// WithTableLoadingMode returns a new Options value with TableLoadingMode set to the given value.
|
|
//
|
|
// TableLoadingMode indicates which file loading mode should be used for the LSM tree data files.
|
|
//
|
|
// The default value of TableLoadingMode is options.MemoryMap.
|
|
func (opt Options) WithTableLoadingMode(val options.FileLoadingMode) Options {
|
|
opt.TableLoadingMode = val
|
|
return opt
|
|
}
|
|
|
|
// WithValueLogLoadingMode returns a new Options value with ValueLogLoadingMode set to the given
|
|
// value.
|
|
//
|
|
// ValueLogLoadingMode indicates which file loading mode should be used for the value log data
|
|
// files.
|
|
//
|
|
// The default value of ValueLogLoadingMode is options.MemoryMap.
|
|
func (opt Options) WithValueLogLoadingMode(val options.FileLoadingMode) Options {
|
|
opt.ValueLogLoadingMode = val
|
|
return opt
|
|
}
|
|
|
|
// WithNumVersionsToKeep returns a new Options value with NumVersionsToKeep set to the given value.
|
|
//
|
|
// NumVersionsToKeep sets how many versions to keep per key at most.
|
|
//
|
|
// The default value of NumVersionsToKeep is 1.
|
|
func (opt Options) WithNumVersionsToKeep(val int) Options {
|
|
opt.NumVersionsToKeep = val
|
|
return opt
|
|
}
|
|
|
|
// WithReadOnly returns a new Options value with ReadOnly set to the given value.
|
|
//
|
|
// When ReadOnly is true the DB will be opened on read-only mode.
|
|
// Multiple processes can open the same Badger DB.
|
|
// Note: if the DB being opened had crashed before and has vlog data to be replayed,
|
|
// ReadOnly will cause Open to fail with an appropriate message.
|
|
//
|
|
// The default value of ReadOnly is false.
|
|
func (opt Options) WithReadOnly(val bool) Options {
|
|
opt.ReadOnly = val
|
|
return opt
|
|
}
|
|
|
|
// WithTruncate returns a new Options value with Truncate set to the given value.
|
|
//
|
|
// Truncate indicates whether value log files should be truncated to delete corrupt data, if any.
|
|
// This option is ignored when ReadOnly is true.
|
|
//
|
|
// The default value of Truncate is false.
|
|
func (opt Options) WithTruncate(val bool) Options {
|
|
opt.Truncate = val
|
|
return opt
|
|
}
|
|
|
|
// WithLogger returns a new Options value with Logger set to the given value.
|
|
//
|
|
// Logger provides a way to configure what logger each value of badger.DB uses.
|
|
//
|
|
// The default value of Logger writes to stderr using the log package from the Go standard library.
|
|
func (opt Options) WithLogger(val Logger) Options {
|
|
opt.Logger = val
|
|
return opt
|
|
}
|
|
|
|
// WithEventLogging returns a new Options value with EventLogging set to the given value.
|
|
//
|
|
// EventLogging provides a way to enable or disable trace.EventLog logging.
|
|
//
|
|
// The default value of EventLogging is true.
|
|
func (opt Options) WithEventLogging(enabled bool) Options {
|
|
opt.EventLogging = enabled
|
|
return opt
|
|
}
|
|
|
|
// WithMaxTableSize returns a new Options value with MaxTableSize set to the given value.
|
|
//
|
|
// MaxTableSize sets the maximum size in bytes for each LSM table or file.
|
|
//
|
|
// The default value of MaxTableSize is 64MB.
|
|
func (opt Options) WithMaxTableSize(val int64) Options {
|
|
opt.MaxTableSize = val
|
|
return opt
|
|
}
|
|
|
|
// WithLevelSizeMultiplier returns a new Options value with LevelSizeMultiplier set to the given
|
|
// value.
|
|
//
|
|
// LevelSizeMultiplier sets the ratio between the maximum sizes of contiguous levels in the LSM.
|
|
// Once a level grows to be larger than this ratio allowed, the compaction process will be
|
|
// triggered.
|
|
//
|
|
// The default value of LevelSizeMultiplier is 10.
|
|
func (opt Options) WithLevelSizeMultiplier(val int) Options {
|
|
opt.LevelSizeMultiplier = val
|
|
return opt
|
|
}
|
|
|
|
// WithMaxLevels returns a new Options value with MaxLevels set to the given value.
|
|
//
|
|
// Maximum number of levels of compaction allowed in the LSM.
|
|
//
|
|
// The default value of MaxLevels is 7.
|
|
func (opt Options) WithMaxLevels(val int) Options {
|
|
opt.MaxLevels = val
|
|
return opt
|
|
}
|
|
|
|
// WithValueThreshold returns a new Options value with ValueThreshold set to the given value.
|
|
//
|
|
// ValueThreshold sets the threshold used to decide whether a value is stored directly in the LSM
|
|
// tree or separatedly in the log value files.
|
|
//
|
|
// The default value of ValueThreshold is 32, but LSMOnlyOptions sets it to 65500.
|
|
func (opt Options) WithValueThreshold(val int) Options {
|
|
opt.ValueThreshold = val
|
|
return opt
|
|
}
|
|
|
|
// WithNumMemtables returns a new Options value with NumMemtables set to the given value.
|
|
//
|
|
// NumMemtables sets the maximum number of tables to keep in memory before stalling.
|
|
//
|
|
// The default value of NumMemtables is 5.
|
|
func (opt Options) WithNumMemtables(val int) Options {
|
|
opt.NumMemtables = val
|
|
return opt
|
|
}
|
|
|
|
// WithNumLevelZeroTables returns a new Options value with NumLevelZeroTables set to the given
|
|
// value.
|
|
//
|
|
// NumLevelZeroTables sets the maximum number of Level 0 tables before compaction starts.
|
|
//
|
|
// The default value of NumLevelZeroTables is 5.
|
|
func (opt Options) WithNumLevelZeroTables(val int) Options {
|
|
opt.NumLevelZeroTables = val
|
|
return opt
|
|
}
|
|
|
|
// WithNumLevelZeroTablesStall returns a new Options value with NumLevelZeroTablesStall set to the
|
|
// given value.
|
|
//
|
|
// NumLevelZeroTablesStall sets the number of Level 0 tables that once reached causes the DB to
|
|
// stall until compaction succeeds.
|
|
//
|
|
// The default value of NumLevelZeroTablesStall is 10.
|
|
func (opt Options) WithNumLevelZeroTablesStall(val int) Options {
|
|
opt.NumLevelZeroTablesStall = val
|
|
return opt
|
|
}
|
|
|
|
// WithLevelOneSize returns a new Options value with LevelOneSize set to the given value.
|
|
//
|
|
// LevelOneSize sets the maximum total size for Level 1.
|
|
//
|
|
// The default value of LevelOneSize is 20MB.
|
|
func (opt Options) WithLevelOneSize(val int64) Options {
|
|
opt.LevelOneSize = val
|
|
return opt
|
|
}
|
|
|
|
// WithValueLogFileSize returns a new Options value with ValueLogFileSize set to the given value.
|
|
//
|
|
// ValueLogFileSize sets the maximum size of a single value log file.
|
|
//
|
|
// The default value of ValueLogFileSize is 1GB.
|
|
func (opt Options) WithValueLogFileSize(val int64) Options {
|
|
opt.ValueLogFileSize = val
|
|
return opt
|
|
}
|
|
|
|
// WithValueLogMaxEntries returns a new Options value with ValueLogMaxEntries set to the given
|
|
// value.
|
|
//
|
|
// ValueLogMaxEntries sets the maximum number of entries a value log file can hold approximately.
|
|
// A actual size limit of a value log file is the minimum of ValueLogFileSize and
|
|
// ValueLogMaxEntries.
|
|
//
|
|
// The default value of ValueLogMaxEntries is one million (1000000).
|
|
func (opt Options) WithValueLogMaxEntries(val uint32) Options {
|
|
opt.ValueLogMaxEntries = val
|
|
return opt
|
|
}
|
|
|
|
// WithNumCompactors returns a new Options value with NumCompactors set to the given value.
|
|
//
|
|
// NumCompactors sets the number of compaction workers to run concurrently.
|
|
// Setting this to zero stops compactions, which could eventually cause writes to block forever.
|
|
//
|
|
// The default value of NumCompactors is 2.
|
|
func (opt Options) WithNumCompactors(val int) Options {
|
|
opt.NumCompactors = val
|
|
return opt
|
|
}
|
|
|
|
// WithCompactL0OnClose returns a new Options value with CompactL0OnClose set to the given value.
|
|
//
|
|
// CompactL0OnClose determines whether Level 0 should be compacted before closing the DB.
|
|
// This ensures that both reads and writes are efficient when the DB is opened later.
|
|
//
|
|
// The default value of CompactL0OnClose is true.
|
|
func (opt Options) WithCompactL0OnClose(val bool) Options {
|
|
opt.CompactL0OnClose = val
|
|
return opt
|
|
}
|
|
|
|
// WithLogRotatesToFlush returns a new Options value with LogRotatesToFlush set to the given value.
|
|
//
|
|
// LogRotatesToFlush sets the number of value log file rotates after which the Memtables are
|
|
// flushed to disk. This is useful in write loads with fewer keys and larger values. This work load
|
|
// would fill up the value logs quickly, while not filling up the Memtables. Thus, on a crash
|
|
// and restart, the value log head could cause the replay of a good number of value log files
|
|
// which can slow things on start.
|
|
//
|
|
// The default value of LogRotatesToFlush is 2.
|
|
func (opt Options) WithLogRotatesToFlush(val int32) Options {
|
|
opt.LogRotatesToFlush = val
|
|
return opt
|
|
}
|
|
|
|
// WithVerifyValueChecksum returns a new Options value with VerifyValueChecksum set to
|
|
// the given value.
|
|
//
|
|
// When VerifyValueChecksum is set to true, checksum will be verified for every entry read
|
|
// from the value log. If the value is stored in SST (value size less than value threshold) then the
|
|
// checksum validation will not be done.
|
|
//
|
|
// The default value of VerifyValueChecksum is False.
|
|
func (opt Options) WithVerifyValueChecksum(val bool) Options {
|
|
opt.VerifyValueChecksum = val
|
|
return opt
|
|
}
|
|
|
|
// WithBypassLockGuard returns a new Options value with BypassLockGuard
|
|
// set to the given value.
|
|
//
|
|
// When BypassLockGuard option is set, badger will not acquire a lock on the
|
|
// directory. This could lead to data corruption if multiple badger instances
|
|
// write to the same data directory. Use this option with caution.
|
|
//
|
|
// The default value of BypassLockGuard is false.
|
|
func (opt Options) WithBypassLockGuard(b bool) Options {
|
|
opt.BypassLockGuard = b
|
|
return opt
|
|
}
|