Add pushover notifications, this should be a super basic MVP

This commit is contained in:
Tony Blyler 2021-05-18 09:04:15 -04:00
parent ed13a5994f
commit d9917ab8b0
Signed by: tblyler
GPG key ID: 7F13D9A60C0D678E
505 changed files with 195741 additions and 9 deletions

83
vendor/github.com/dgraph-io/badger/y/error.go generated vendored Normal file
View file

@ -0,0 +1,83 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package y
// This file contains some functions for error handling. Note that we are moving
// towards using x.Trace, i.e., rpc tracing using net/tracer. But for now, these
// functions are useful for simple checks logged on one machine.
// Some common use cases are:
// (1) You receive an error from external lib, and would like to check/log fatal.
// For this, use x.Check, x.Checkf. These will check for err != nil, which is
// more common in Go. If you want to check for boolean being true, use
// x.Assert, x.Assertf.
// (2) You receive an error from external lib, and would like to pass on with some
// stack trace information. In this case, use x.Wrap or x.Wrapf.
// (3) You want to generate a new error with stack trace info. Use x.Errorf.
import (
"fmt"
"log"
"github.com/pkg/errors"
)
var debugMode = true
// Check logs fatal if err != nil.
func Check(err error) {
if err != nil {
log.Fatalf("%+v", Wrap(err))
}
}
// Check2 acts as convenience wrapper around Check, using the 2nd argument as error.
func Check2(_ interface{}, err error) {
Check(err)
}
// AssertTrue asserts that b is true. Otherwise, it would log fatal.
func AssertTrue(b bool) {
if !b {
log.Fatalf("%+v", errors.Errorf("Assert failed"))
}
}
// AssertTruef is AssertTrue with extra info.
func AssertTruef(b bool, format string, args ...interface{}) {
if !b {
log.Fatalf("%+v", errors.Errorf(format, args...))
}
}
// Wrap wraps errors from external lib.
func Wrap(err error) error {
if !debugMode {
return err
}
return errors.Wrap(err, "")
}
// Wrapf is Wrap with extra info.
func Wrapf(err error, format string, args ...interface{}) error {
if !debugMode {
if err == nil {
return nil
}
return fmt.Errorf(format+" error: %+v", append(args, err)...)
}
return errors.Wrapf(err, format, args...)
}

31
vendor/github.com/dgraph-io/badger/y/event_log.go generated vendored Normal file
View file

@ -0,0 +1,31 @@
/*
* Copyright 2019 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package y
import "golang.org/x/net/trace"
var (
NoEventLog trace.EventLog = nilEventLog{}
)
type nilEventLog struct{}
func (nel nilEventLog) Printf(format string, a ...interface{}) {}
func (nel nilEventLog) Errorf(format string, a ...interface{}) {}
func (nel nilEventLog) Finish() {}

25
vendor/github.com/dgraph-io/badger/y/file_dsync.go generated vendored Normal file
View file

@ -0,0 +1,25 @@
// +build !dragonfly,!freebsd,!windows,!plan9
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package y
import "golang.org/x/sys/unix"
func init() {
datasyncFileFlag = unix.O_DSYNC
}

25
vendor/github.com/dgraph-io/badger/y/file_nodsync.go generated vendored Normal file
View file

@ -0,0 +1,25 @@
// +build dragonfly freebsd windows plan9
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package y
import "syscall"
func init() {
datasyncFileFlag = syscall.O_SYNC
}

28
vendor/github.com/dgraph-io/badger/y/file_sync.go generated vendored Normal file
View file

@ -0,0 +1,28 @@
// +build !darwin go1.12
/*
* Copyright 2019 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package y
import "os"
// FileSync calls os.File.Sync with the right parameters.
// This function can be removed once we stop supporting Go 1.11
// on MacOS.
//
// More info: https://golang.org/issue/26650.
func FileSync(f *os.File) error { return f.Sync() }

View file

@ -0,0 +1,37 @@
// +build darwin,!go1.12
/*
* Copyright 2019 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package y
import (
"os"
"syscall"
)
// FileSync calls os.File.Sync with the right parameters.
// This function can be removed once we stop supporting Go 1.11
// on MacOS.
//
// More info: https://golang.org/issue/26650.
func FileSync(f *os.File) error {
_, _, err := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), syscall.F_FULLFSYNC, 0)
if err == 0 {
return nil
}
return err
}

97
vendor/github.com/dgraph-io/badger/y/iterator.go generated vendored Normal file
View file

@ -0,0 +1,97 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package y
import (
"bytes"
"encoding/binary"
)
// ValueStruct represents the value info that can be associated with a key, but also the internal
// Meta field.
type ValueStruct struct {
Meta byte
UserMeta byte
ExpiresAt uint64
Value []byte
Version uint64 // This field is not serialized. Only for internal usage.
}
func sizeVarint(x uint64) (n int) {
for {
n++
x >>= 7
if x == 0 {
break
}
}
return n
}
// EncodedSize is the size of the ValueStruct when encoded
func (v *ValueStruct) EncodedSize() uint16 {
sz := len(v.Value) + 2 // meta, usermeta.
if v.ExpiresAt == 0 {
return uint16(sz + 1)
}
enc := sizeVarint(v.ExpiresAt)
return uint16(sz + enc)
}
// Decode uses the length of the slice to infer the length of the Value field.
func (v *ValueStruct) Decode(b []byte) {
v.Meta = b[0]
v.UserMeta = b[1]
var sz int
v.ExpiresAt, sz = binary.Uvarint(b[2:])
v.Value = b[2+sz:]
}
// Encode expects a slice of length at least v.EncodedSize().
func (v *ValueStruct) Encode(b []byte) {
b[0] = v.Meta
b[1] = v.UserMeta
sz := binary.PutUvarint(b[2:], v.ExpiresAt)
copy(b[2+sz:], v.Value)
}
// EncodeTo should be kept in sync with the Encode function above. The reason
// this function exists is to avoid creating byte arrays per key-value pair in
// table/builder.go.
func (v *ValueStruct) EncodeTo(buf *bytes.Buffer) {
buf.WriteByte(v.Meta)
buf.WriteByte(v.UserMeta)
var enc [binary.MaxVarintLen64]byte
sz := binary.PutUvarint(enc[:], v.ExpiresAt)
buf.Write(enc[:sz])
buf.Write(v.Value)
}
// Iterator is an interface for a basic iterator.
type Iterator interface {
Next()
Rewind()
Seek(key []byte)
Key() []byte
Value() ValueStruct
Valid() bool
// All iterators should be closed so that file garbage collection works.
Close() error
}

68
vendor/github.com/dgraph-io/badger/y/metrics.go generated vendored Normal file
View file

@ -0,0 +1,68 @@
/*
* Copyright (C) 2017 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package y
import "expvar"
var (
// LSMSize has size of the LSM in bytes
LSMSize *expvar.Map
// VlogSize has size of the value log in bytes
VlogSize *expvar.Map
// PendingWrites tracks the number of pending writes.
PendingWrites *expvar.Map
// These are cumulative
// NumReads has cumulative number of reads
NumReads *expvar.Int
// NumWrites has cumulative number of writes
NumWrites *expvar.Int
// NumBytesRead has cumulative number of bytes read
NumBytesRead *expvar.Int
// NumBytesWritten has cumulative number of bytes written
NumBytesWritten *expvar.Int
// NumLSMGets is number of LMS gets
NumLSMGets *expvar.Map
// NumLSMBloomHits is number of LMS bloom hits
NumLSMBloomHits *expvar.Map
// NumGets is number of gets
NumGets *expvar.Int
// NumPuts is number of puts
NumPuts *expvar.Int
// NumBlockedPuts is number of blocked puts
NumBlockedPuts *expvar.Int
// NumMemtableGets is number of memtable gets
NumMemtableGets *expvar.Int
)
// These variables are global and have cumulative values for all kv stores.
func init() {
NumReads = expvar.NewInt("badger_disk_reads_total")
NumWrites = expvar.NewInt("badger_disk_writes_total")
NumBytesRead = expvar.NewInt("badger_read_bytes")
NumBytesWritten = expvar.NewInt("badger_written_bytes")
NumLSMGets = expvar.NewMap("badger_lsm_level_gets_total")
NumLSMBloomHits = expvar.NewMap("badger_lsm_bloom_hits_total")
NumGets = expvar.NewInt("badger_gets_total")
NumPuts = expvar.NewInt("badger_puts_total")
NumBlockedPuts = expvar.NewInt("badger_blocked_puts_total")
NumMemtableGets = expvar.NewInt("badger_memtable_gets_total")
LSMSize = expvar.NewMap("badger_lsm_size_bytes")
VlogSize = expvar.NewMap("badger_vlog_size_bytes")
PendingWrites = expvar.NewMap("badger_pending_writes_total")
}

39
vendor/github.com/dgraph-io/badger/y/mmap.go generated vendored Normal file
View file

@ -0,0 +1,39 @@
/*
* Copyright 2019 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package y
import (
"os"
)
// Mmap uses the mmap system call to memory-map a file. If writable is true,
// memory protection of the pages is set so that they may be written to as well.
func Mmap(fd *os.File, writable bool, size int64) ([]byte, error) {
return mmap(fd, writable, size)
}
// Munmap unmaps a previously mapped slice.
func Munmap(b []byte) error {
return munmap(b)
}
// Madvise uses the madvise system call to give advise about the use of memory
// when using a slice that is memory-mapped to a file. Set the readahead flag to
// false if page references are expected in random order.
func Madvise(b []byte, readahead bool) error {
return madvise(b, readahead)
}

55
vendor/github.com/dgraph-io/badger/y/mmap_darwin.go generated vendored Normal file
View file

@ -0,0 +1,55 @@
/*
* Copyright 2019 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package y
import (
"os"
"syscall"
"unsafe"
"golang.org/x/sys/unix"
)
// Mmap uses the mmap system call to memory-map a file. If writable is true,
// memory protection of the pages is set so that they may be written to as well.
func mmap(fd *os.File, writable bool, size int64) ([]byte, error) {
mtype := unix.PROT_READ
if writable {
mtype |= unix.PROT_WRITE
}
return unix.Mmap(int(fd.Fd()), 0, int(size), mtype, unix.MAP_SHARED)
}
// Munmap unmaps a previously mapped slice.
func munmap(b []byte) error {
return unix.Munmap(b)
}
// This is required because the unix package does not support the madvise system call on OS X.
func madvise(b []byte, readahead bool) error {
advice := unix.MADV_NORMAL
if !readahead {
advice = unix.MADV_RANDOM
}
_, _, e1 := syscall.Syscall(syscall.SYS_MADVISE, uintptr(unsafe.Pointer(&b[0])),
uintptr(len(b)), uintptr(advice))
if e1 != 0 {
return e1
}
return nil
}

40
vendor/github.com/dgraph-io/badger/y/mmap_plan9.go generated vendored Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright 2020 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package y
import (
"os"
"syscall"
)
// Mmap uses the mmap system call to memory-map a file. If writable is true,
// memory protection of the pages is set so that they may be written to as well.
func mmap(fd *os.File, writable bool, size int64) ([]byte, error) {
return nil, syscall.EPLAN9
}
// Munmap unmaps a previously mapped slice.
func munmap(b []byte) error {
return syscall.EPLAN9
}
// Madvise uses the madvise system call to give advise about the use of memory
// when using a slice that is memory-mapped to a file. Set the readahead flag to
// false if page references are expected in random order.
func madvise(b []byte, readahead bool) error {
return syscall.EPLAN9
}

51
vendor/github.com/dgraph-io/badger/y/mmap_unix.go generated vendored Normal file
View file

@ -0,0 +1,51 @@
// +build !windows,!darwin,!plan9
/*
* Copyright 2019 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package y
import (
"os"
"golang.org/x/sys/unix"
)
// Mmap uses the mmap system call to memory-map a file. If writable is true,
// memory protection of the pages is set so that they may be written to as well.
func mmap(fd *os.File, writable bool, size int64) ([]byte, error) {
mtype := unix.PROT_READ
if writable {
mtype |= unix.PROT_WRITE
}
return unix.Mmap(int(fd.Fd()), 0, int(size), mtype, unix.MAP_SHARED)
}
// Munmap unmaps a previously mapped slice.
func munmap(b []byte) error {
return unix.Munmap(b)
}
// Madvise uses the madvise system call to give advise about the use of memory
// when using a slice that is memory-mapped to a file. Set the readahead flag to
// false if page references are expected in random order.
func madvise(b []byte, readahead bool) error {
flags := unix.MADV_NORMAL
if !readahead {
flags = unix.MADV_RANDOM
}
return unix.Madvise(b, flags)
}

91
vendor/github.com/dgraph-io/badger/y/mmap_windows.go generated vendored Normal file
View file

@ -0,0 +1,91 @@
// +build windows
/*
* Copyright 2019 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package y
import (
"fmt"
"os"
"syscall"
"unsafe"
)
func mmap(fd *os.File, write bool, size int64) ([]byte, error) {
protect := syscall.PAGE_READONLY
access := syscall.FILE_MAP_READ
if write {
protect = syscall.PAGE_READWRITE
access = syscall.FILE_MAP_WRITE
}
fi, err := fd.Stat()
if err != nil {
return nil, err
}
// In windows, we cannot mmap a file more than it's actual size.
// So truncate the file to the size of the mmap.
if fi.Size() < size {
if err := fd.Truncate(size); err != nil {
return nil, fmt.Errorf("truncate: %s", err)
}
}
// Open a file mapping handle.
sizelo := uint32(size >> 32)
sizehi := uint32(size) & 0xffffffff
handler, err := syscall.CreateFileMapping(syscall.Handle(fd.Fd()), nil,
uint32(protect), sizelo, sizehi, nil)
if err != nil {
return nil, os.NewSyscallError("CreateFileMapping", err)
}
// Create the memory map.
addr, err := syscall.MapViewOfFile(handler, uint32(access), 0, 0, uintptr(size))
if addr == 0 {
return nil, os.NewSyscallError("MapViewOfFile", err)
}
// Close mapping handle.
if err := syscall.CloseHandle(syscall.Handle(handler)); err != nil {
return nil, os.NewSyscallError("CloseHandle", err)
}
// Slice memory layout
// Copied this snippet from golang/sys package
var sl = struct {
addr uintptr
len int
cap int
}{addr, int(size), int(size)}
// Use unsafe to turn sl into a []byte.
data := *(*[]byte)(unsafe.Pointer(&sl))
return data, nil
}
func munmap(b []byte) error {
return syscall.UnmapViewOfFile(uintptr(unsafe.Pointer(&b[0])))
}
func madvise(b []byte, readahead bool) error {
// Do Nothing. We dont care about this setting on Windows
return nil
}

255
vendor/github.com/dgraph-io/badger/y/watermark.go generated vendored Normal file
View file

@ -0,0 +1,255 @@
/*
* Copyright 2016-2018 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package y
import (
"container/heap"
"context"
"sync/atomic"
"golang.org/x/net/trace"
)
type uint64Heap []uint64
func (u uint64Heap) Len() int { return len(u) }
func (u uint64Heap) Less(i, j int) bool { return u[i] < u[j] }
func (u uint64Heap) Swap(i, j int) { u[i], u[j] = u[j], u[i] }
func (u *uint64Heap) Push(x interface{}) { *u = append(*u, x.(uint64)) }
func (u *uint64Heap) Pop() interface{} {
old := *u
n := len(old)
x := old[n-1]
*u = old[0 : n-1]
return x
}
// mark contains one of more indices, along with a done boolean to indicate the
// status of the index: begin or done. It also contains waiters, who could be
// waiting for the watermark to reach >= a certain index.
type mark struct {
// Either this is an (index, waiter) pair or (index, done) or (indices, done).
index uint64
waiter chan struct{}
indices []uint64
done bool // Set to true if the index is done.
}
// WaterMark is used to keep track of the minimum un-finished index. Typically, an index k becomes
// finished or "done" according to a WaterMark once Done(k) has been called
// 1. as many times as Begin(k) has, AND
// 2. a positive number of times.
//
// An index may also become "done" by calling SetDoneUntil at a time such that it is not
// inter-mingled with Begin/Done calls.
//
// Since doneUntil and lastIndex addresses are passed to sync/atomic packages, we ensure that they
// are 64-bit aligned by putting them at the beginning of the structure.
type WaterMark struct {
doneUntil uint64
lastIndex uint64
Name string
markCh chan mark
elog trace.EventLog
}
// Init initializes a WaterMark struct. MUST be called before using it.
func (w *WaterMark) Init(closer *Closer, eventLogging bool) {
w.markCh = make(chan mark, 100)
if eventLogging {
w.elog = trace.NewEventLog("Watermark", w.Name)
} else {
w.elog = NoEventLog
}
go w.process(closer)
}
// Begin sets the last index to the given value.
func (w *WaterMark) Begin(index uint64) {
atomic.StoreUint64(&w.lastIndex, index)
w.markCh <- mark{index: index, done: false}
}
// BeginMany works like Begin but accepts multiple indices.
func (w *WaterMark) BeginMany(indices []uint64) {
atomic.StoreUint64(&w.lastIndex, indices[len(indices)-1])
w.markCh <- mark{index: 0, indices: indices, done: false}
}
// Done sets a single index as done.
func (w *WaterMark) Done(index uint64) {
w.markCh <- mark{index: index, done: true}
}
// DoneMany works like Done but accepts multiple indices.
func (w *WaterMark) DoneMany(indices []uint64) {
w.markCh <- mark{index: 0, indices: indices, done: true}
}
// DoneUntil returns the maximum index that has the property that all indices
// less than or equal to it are done.
func (w *WaterMark) DoneUntil() uint64 {
return atomic.LoadUint64(&w.doneUntil)
}
// SetDoneUntil sets the maximum index that has the property that all indices
// less than or equal to it are done.
func (w *WaterMark) SetDoneUntil(val uint64) {
atomic.StoreUint64(&w.doneUntil, val)
}
// LastIndex returns the last index for which Begin has been called.
func (w *WaterMark) LastIndex() uint64 {
return atomic.LoadUint64(&w.lastIndex)
}
// WaitForMark waits until the given index is marked as done.
func (w *WaterMark) WaitForMark(ctx context.Context, index uint64) error {
if w.DoneUntil() >= index {
return nil
}
waitCh := make(chan struct{})
w.markCh <- mark{index: index, waiter: waitCh}
select {
case <-ctx.Done():
return ctx.Err()
case <-waitCh:
return nil
}
}
// process is used to process the Mark channel. This is not thread-safe,
// so only run one goroutine for process. One is sufficient, because
// all goroutine ops use purely memory and cpu.
// Each index has to emit atleast one begin watermark in serial order otherwise waiters
// can get blocked idefinitely. Example: We had an watermark at 100 and a waiter at 101,
// if no watermark is emitted at index 101 then waiter would get stuck indefinitely as it
// can't decide whether the task at 101 has decided not to emit watermark or it didn't get
// scheduled yet.
func (w *WaterMark) process(closer *Closer) {
defer closer.Done()
var indices uint64Heap
// pending maps raft proposal index to the number of pending mutations for this proposal.
pending := make(map[uint64]int)
waiters := make(map[uint64][]chan struct{})
heap.Init(&indices)
var loop uint64
processOne := func(index uint64, done bool) {
// If not already done, then set. Otherwise, don't undo a done entry.
prev, present := pending[index]
if !present {
heap.Push(&indices, index)
}
delta := 1
if done {
delta = -1
}
pending[index] = prev + delta
loop++
if len(indices) > 0 && loop%10000 == 0 {
min := indices[0]
w.elog.Printf("WaterMark %s: Done entry %4d. Size: %4d Watermark: %-4d Looking for: "+
"%-4d. Value: %d\n", w.Name, index, len(indices), w.DoneUntil(), min, pending[min])
}
// Update mark by going through all indices in order; and checking if they have
// been done. Stop at the first index, which isn't done.
doneUntil := w.DoneUntil()
if doneUntil > index {
AssertTruef(false, "Name: %s doneUntil: %d. Index: %d", w.Name, doneUntil, index)
}
until := doneUntil
loops := 0
for len(indices) > 0 {
min := indices[0]
if done := pending[min]; done > 0 {
break // len(indices) will be > 0.
}
// Even if done is called multiple times causing it to become
// negative, we should still pop the index.
heap.Pop(&indices)
delete(pending, min)
until = min
loops++
}
if until != doneUntil {
AssertTrue(atomic.CompareAndSwapUint64(&w.doneUntil, doneUntil, until))
w.elog.Printf("%s: Done until %d. Loops: %d\n", w.Name, until, loops)
}
notifyAndRemove := func(idx uint64, toNotify []chan struct{}) {
for _, ch := range toNotify {
close(ch)
}
delete(waiters, idx) // Release the memory back.
}
if until-doneUntil <= uint64(len(waiters)) {
// Issue #908 showed that if doneUntil is close to 2^60, while until is zero, this loop
// can hog up CPU just iterating over integers creating a busy-wait loop. So, only do
// this path if until - doneUntil is less than the number of waiters.
for idx := doneUntil + 1; idx <= until; idx++ {
if toNotify, ok := waiters[idx]; ok {
notifyAndRemove(idx, toNotify)
}
}
} else {
for idx, toNotify := range waiters {
if idx <= until {
notifyAndRemove(idx, toNotify)
}
}
} // end of notifying waiters.
}
for {
select {
case <-closer.HasBeenClosed():
return
case mark := <-w.markCh:
if mark.waiter != nil {
doneUntil := atomic.LoadUint64(&w.doneUntil)
if doneUntil >= mark.index {
close(mark.waiter)
} else {
ws, ok := waiters[mark.index]
if !ok {
waiters[mark.index] = []chan struct{}{mark.waiter}
} else {
waiters[mark.index] = append(ws, mark.waiter)
}
}
} else {
if mark.index > 0 {
processOne(mark.index, mark.done)
}
for _, index := range mark.indices {
processOne(index, mark.done)
}
}
}
}
}

302
vendor/github.com/dgraph-io/badger/y/y.go generated vendored Normal file
View file

@ -0,0 +1,302 @@
/*
* Copyright 2017 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package y
import (
"bytes"
"encoding/binary"
"fmt"
"hash/crc32"
"math"
"os"
"sync"
"time"
"github.com/pkg/errors"
)
// ErrEOF indicates an end of file when trying to read from a memory mapped file
// and encountering the end of slice.
var ErrEOF = errors.New("End of mapped region")
const (
// Sync indicates that O_DSYNC should be set on the underlying file,
// ensuring that data writes do not return until the data is flushed
// to disk.
Sync = 1 << iota
// ReadOnly opens the underlying file on a read-only basis.
ReadOnly
)
var (
// This is O_DSYNC (datasync) on platforms that support it -- see file_unix.go
datasyncFileFlag = 0x0
// CastagnoliCrcTable is a CRC32 polynomial table
CastagnoliCrcTable = crc32.MakeTable(crc32.Castagnoli)
// Dummy channel for nil closers.
dummyCloserChan = make(chan struct{})
)
// OpenExistingFile opens an existing file, errors if it doesn't exist.
func OpenExistingFile(filename string, flags uint32) (*os.File, error) {
openFlags := os.O_RDWR
if flags&ReadOnly != 0 {
openFlags = os.O_RDONLY
}
if flags&Sync != 0 {
openFlags |= datasyncFileFlag
}
return os.OpenFile(filename, openFlags, 0)
}
// CreateSyncedFile creates a new file (using O_EXCL), errors if it already existed.
func CreateSyncedFile(filename string, sync bool) (*os.File, error) {
flags := os.O_RDWR | os.O_CREATE | os.O_EXCL
if sync {
flags |= datasyncFileFlag
}
return os.OpenFile(filename, flags, 0600)
}
// OpenSyncedFile creates the file if one doesn't exist.
func OpenSyncedFile(filename string, sync bool) (*os.File, error) {
flags := os.O_RDWR | os.O_CREATE
if sync {
flags |= datasyncFileFlag
}
return os.OpenFile(filename, flags, 0600)
}
// OpenTruncFile opens the file with O_RDWR | O_CREATE | O_TRUNC
func OpenTruncFile(filename string, sync bool) (*os.File, error) {
flags := os.O_RDWR | os.O_CREATE | os.O_TRUNC
if sync {
flags |= datasyncFileFlag
}
return os.OpenFile(filename, flags, 0600)
}
// SafeCopy does append(a[:0], src...).
func SafeCopy(a, src []byte) []byte {
return append(a[:0], src...)
}
// Copy copies a byte slice and returns the copied slice.
func Copy(a []byte) []byte {
b := make([]byte, len(a))
copy(b, a)
return b
}
// KeyWithTs generates a new key by appending ts to key.
func KeyWithTs(key []byte, ts uint64) []byte {
out := make([]byte, len(key)+8)
copy(out, key)
binary.BigEndian.PutUint64(out[len(key):], math.MaxUint64-ts)
return out
}
// ParseTs parses the timestamp from the key bytes.
func ParseTs(key []byte) uint64 {
if len(key) <= 8 {
return 0
}
return math.MaxUint64 - binary.BigEndian.Uint64(key[len(key)-8:])
}
// CompareKeys checks the key without timestamp and checks the timestamp if keyNoTs
// is same.
// a<timestamp> would be sorted higher than aa<timestamp> if we use bytes.compare
// All keys should have timestamp.
func CompareKeys(key1, key2 []byte) int {
AssertTrue(len(key1) > 8 && len(key2) > 8)
if cmp := bytes.Compare(key1[:len(key1)-8], key2[:len(key2)-8]); cmp != 0 {
return cmp
}
return bytes.Compare(key1[len(key1)-8:], key2[len(key2)-8:])
}
// ParseKey parses the actual key from the key bytes.
func ParseKey(key []byte) []byte {
if key == nil {
return nil
}
AssertTrue(len(key) > 8)
return key[:len(key)-8]
}
// SameKey checks for key equality ignoring the version timestamp suffix.
func SameKey(src, dst []byte) bool {
if len(src) != len(dst) {
return false
}
return bytes.Equal(ParseKey(src), ParseKey(dst))
}
// Slice holds a reusable buf, will reallocate if you request a larger size than ever before.
// One problem is with n distinct sizes in random order it'll reallocate log(n) times.
type Slice struct {
buf []byte
}
// Resize reuses the Slice's buffer (or makes a new one) and returns a slice in that buffer of
// length sz.
func (s *Slice) Resize(sz int) []byte {
if cap(s.buf) < sz {
s.buf = make([]byte, sz)
}
return s.buf[0:sz]
}
// FixedDuration returns a string representation of the given duration with the
// hours, minutes, and seconds.
func FixedDuration(d time.Duration) string {
str := fmt.Sprintf("%02ds", int(d.Seconds())%60)
if d >= time.Minute {
str = fmt.Sprintf("%02dm", int(d.Minutes())%60) + str
}
if d >= time.Hour {
str = fmt.Sprintf("%02dh", int(d.Hours())) + str
}
return str
}
// Closer holds the two things we need to close a goroutine and wait for it to finish: a chan
// to tell the goroutine to shut down, and a WaitGroup with which to wait for it to finish shutting
// down.
type Closer struct {
closed chan struct{}
waiting sync.WaitGroup
}
// NewCloser constructs a new Closer, with an initial count on the WaitGroup.
func NewCloser(initial int) *Closer {
ret := &Closer{closed: make(chan struct{})}
ret.waiting.Add(initial)
return ret
}
// AddRunning Add()'s delta to the WaitGroup.
func (lc *Closer) AddRunning(delta int) {
lc.waiting.Add(delta)
}
// Signal signals the HasBeenClosed signal.
func (lc *Closer) Signal() {
close(lc.closed)
}
// HasBeenClosed gets signaled when Signal() is called.
func (lc *Closer) HasBeenClosed() <-chan struct{} {
if lc == nil {
return dummyCloserChan
}
return lc.closed
}
// Done calls Done() on the WaitGroup.
func (lc *Closer) Done() {
if lc == nil {
return
}
lc.waiting.Done()
}
// Wait waits on the WaitGroup. (It waits for NewCloser's initial value, AddRunning, and Done
// calls to balance out.)
func (lc *Closer) Wait() {
lc.waiting.Wait()
}
// SignalAndWait calls Signal(), then Wait().
func (lc *Closer) SignalAndWait() {
lc.Signal()
lc.Wait()
}
// Throttle allows a limited number of workers to run at a time. It also
// provides a mechanism to check for errors encountered by workers and wait for
// them to finish.
type Throttle struct {
once sync.Once
wg sync.WaitGroup
ch chan struct{}
errCh chan error
finishErr error
}
// NewThrottle creates a new throttle with a max number of workers.
func NewThrottle(max int) *Throttle {
return &Throttle{
ch: make(chan struct{}, max),
errCh: make(chan error, max),
}
}
// Do should be called by workers before they start working. It blocks if there
// are already maximum number of workers working. If it detects an error from
// previously Done workers, it would return it.
func (t *Throttle) Do() error {
for {
select {
case t.ch <- struct{}{}:
t.wg.Add(1)
return nil
case err := <-t.errCh:
if err != nil {
return err
}
}
}
}
// Done should be called by workers when they finish working. They can also
// pass the error status of work done.
func (t *Throttle) Done(err error) {
if err != nil {
t.errCh <- err
}
select {
case <-t.ch:
default:
panic("Throttle Do Done mismatch")
}
t.wg.Done()
}
// Finish waits until all workers have finished working. It would return any error passed by Done.
// If Finish is called multiple time, it will wait for workers to finish only once(first time).
// From next calls, it will return same error as found on first call.
func (t *Throttle) Finish() error {
t.once.Do(func() {
t.wg.Wait()
close(t.ch)
close(t.errCh)
for err := range t.errCh {
if err != nil {
t.finishErr = err
return
}
}
})
return t.finishErr
}