Add pushover notifications, this should be a super basic MVP
This commit is contained in:
parent
ed13a5994f
commit
d9917ab8b0
505 changed files with 195741 additions and 9 deletions
83
vendor/github.com/dgraph-io/badger/y/error.go
generated
vendored
Normal file
83
vendor/github.com/dgraph-io/badger/y/error.go
generated
vendored
Normal file
|
@ -0,0 +1,83 @@
|
|||
/*
|
||||
* Copyright 2017 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package y
|
||||
|
||||
// This file contains some functions for error handling. Note that we are moving
|
||||
// towards using x.Trace, i.e., rpc tracing using net/tracer. But for now, these
|
||||
// functions are useful for simple checks logged on one machine.
|
||||
// Some common use cases are:
|
||||
// (1) You receive an error from external lib, and would like to check/log fatal.
|
||||
// For this, use x.Check, x.Checkf. These will check for err != nil, which is
|
||||
// more common in Go. If you want to check for boolean being true, use
|
||||
// x.Assert, x.Assertf.
|
||||
// (2) You receive an error from external lib, and would like to pass on with some
|
||||
// stack trace information. In this case, use x.Wrap or x.Wrapf.
|
||||
// (3) You want to generate a new error with stack trace info. Use x.Errorf.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
var debugMode = true
|
||||
|
||||
// Check logs fatal if err != nil.
|
||||
func Check(err error) {
|
||||
if err != nil {
|
||||
log.Fatalf("%+v", Wrap(err))
|
||||
}
|
||||
}
|
||||
|
||||
// Check2 acts as convenience wrapper around Check, using the 2nd argument as error.
|
||||
func Check2(_ interface{}, err error) {
|
||||
Check(err)
|
||||
}
|
||||
|
||||
// AssertTrue asserts that b is true. Otherwise, it would log fatal.
|
||||
func AssertTrue(b bool) {
|
||||
if !b {
|
||||
log.Fatalf("%+v", errors.Errorf("Assert failed"))
|
||||
}
|
||||
}
|
||||
|
||||
// AssertTruef is AssertTrue with extra info.
|
||||
func AssertTruef(b bool, format string, args ...interface{}) {
|
||||
if !b {
|
||||
log.Fatalf("%+v", errors.Errorf(format, args...))
|
||||
}
|
||||
}
|
||||
|
||||
// Wrap wraps errors from external lib.
|
||||
func Wrap(err error) error {
|
||||
if !debugMode {
|
||||
return err
|
||||
}
|
||||
return errors.Wrap(err, "")
|
||||
}
|
||||
|
||||
// Wrapf is Wrap with extra info.
|
||||
func Wrapf(err error, format string, args ...interface{}) error {
|
||||
if !debugMode {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf(format+" error: %+v", append(args, err)...)
|
||||
}
|
||||
return errors.Wrapf(err, format, args...)
|
||||
}
|
31
vendor/github.com/dgraph-io/badger/y/event_log.go
generated
vendored
Normal file
31
vendor/github.com/dgraph-io/badger/y/event_log.go
generated
vendored
Normal file
|
@ -0,0 +1,31 @@
|
|||
/*
|
||||
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package y
|
||||
|
||||
import "golang.org/x/net/trace"
|
||||
|
||||
var (
|
||||
NoEventLog trace.EventLog = nilEventLog{}
|
||||
)
|
||||
|
||||
type nilEventLog struct{}
|
||||
|
||||
func (nel nilEventLog) Printf(format string, a ...interface{}) {}
|
||||
|
||||
func (nel nilEventLog) Errorf(format string, a ...interface{}) {}
|
||||
|
||||
func (nel nilEventLog) Finish() {}
|
25
vendor/github.com/dgraph-io/badger/y/file_dsync.go
generated
vendored
Normal file
25
vendor/github.com/dgraph-io/badger/y/file_dsync.go
generated
vendored
Normal file
|
@ -0,0 +1,25 @@
|
|||
// +build !dragonfly,!freebsd,!windows,!plan9
|
||||
|
||||
/*
|
||||
* Copyright 2017 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package y
|
||||
|
||||
import "golang.org/x/sys/unix"
|
||||
|
||||
func init() {
|
||||
datasyncFileFlag = unix.O_DSYNC
|
||||
}
|
25
vendor/github.com/dgraph-io/badger/y/file_nodsync.go
generated
vendored
Normal file
25
vendor/github.com/dgraph-io/badger/y/file_nodsync.go
generated
vendored
Normal file
|
@ -0,0 +1,25 @@
|
|||
// +build dragonfly freebsd windows plan9
|
||||
|
||||
/*
|
||||
* Copyright 2017 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package y
|
||||
|
||||
import "syscall"
|
||||
|
||||
func init() {
|
||||
datasyncFileFlag = syscall.O_SYNC
|
||||
}
|
28
vendor/github.com/dgraph-io/badger/y/file_sync.go
generated
vendored
Normal file
28
vendor/github.com/dgraph-io/badger/y/file_sync.go
generated
vendored
Normal file
|
@ -0,0 +1,28 @@
|
|||
// +build !darwin go1.12
|
||||
|
||||
/*
|
||||
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package y
|
||||
|
||||
import "os"
|
||||
|
||||
// FileSync calls os.File.Sync with the right parameters.
|
||||
// This function can be removed once we stop supporting Go 1.11
|
||||
// on MacOS.
|
||||
//
|
||||
// More info: https://golang.org/issue/26650.
|
||||
func FileSync(f *os.File) error { return f.Sync() }
|
37
vendor/github.com/dgraph-io/badger/y/file_sync_darwin.go
generated
vendored
Normal file
37
vendor/github.com/dgraph-io/badger/y/file_sync_darwin.go
generated
vendored
Normal file
|
@ -0,0 +1,37 @@
|
|||
// +build darwin,!go1.12
|
||||
|
||||
/*
|
||||
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package y
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// FileSync calls os.File.Sync with the right parameters.
|
||||
// This function can be removed once we stop supporting Go 1.11
|
||||
// on MacOS.
|
||||
//
|
||||
// More info: https://golang.org/issue/26650.
|
||||
func FileSync(f *os.File) error {
|
||||
_, _, err := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), syscall.F_FULLFSYNC, 0)
|
||||
if err == 0 {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
97
vendor/github.com/dgraph-io/badger/y/iterator.go
generated
vendored
Normal file
97
vendor/github.com/dgraph-io/badger/y/iterator.go
generated
vendored
Normal file
|
@ -0,0 +1,97 @@
|
|||
/*
|
||||
* Copyright 2017 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package y
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
// ValueStruct represents the value info that can be associated with a key, but also the internal
|
||||
// Meta field.
|
||||
type ValueStruct struct {
|
||||
Meta byte
|
||||
UserMeta byte
|
||||
ExpiresAt uint64
|
||||
Value []byte
|
||||
|
||||
Version uint64 // This field is not serialized. Only for internal usage.
|
||||
}
|
||||
|
||||
func sizeVarint(x uint64) (n int) {
|
||||
for {
|
||||
n++
|
||||
x >>= 7
|
||||
if x == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// EncodedSize is the size of the ValueStruct when encoded
|
||||
func (v *ValueStruct) EncodedSize() uint16 {
|
||||
sz := len(v.Value) + 2 // meta, usermeta.
|
||||
if v.ExpiresAt == 0 {
|
||||
return uint16(sz + 1)
|
||||
}
|
||||
|
||||
enc := sizeVarint(v.ExpiresAt)
|
||||
return uint16(sz + enc)
|
||||
}
|
||||
|
||||
// Decode uses the length of the slice to infer the length of the Value field.
|
||||
func (v *ValueStruct) Decode(b []byte) {
|
||||
v.Meta = b[0]
|
||||
v.UserMeta = b[1]
|
||||
var sz int
|
||||
v.ExpiresAt, sz = binary.Uvarint(b[2:])
|
||||
v.Value = b[2+sz:]
|
||||
}
|
||||
|
||||
// Encode expects a slice of length at least v.EncodedSize().
|
||||
func (v *ValueStruct) Encode(b []byte) {
|
||||
b[0] = v.Meta
|
||||
b[1] = v.UserMeta
|
||||
sz := binary.PutUvarint(b[2:], v.ExpiresAt)
|
||||
copy(b[2+sz:], v.Value)
|
||||
}
|
||||
|
||||
// EncodeTo should be kept in sync with the Encode function above. The reason
|
||||
// this function exists is to avoid creating byte arrays per key-value pair in
|
||||
// table/builder.go.
|
||||
func (v *ValueStruct) EncodeTo(buf *bytes.Buffer) {
|
||||
buf.WriteByte(v.Meta)
|
||||
buf.WriteByte(v.UserMeta)
|
||||
var enc [binary.MaxVarintLen64]byte
|
||||
sz := binary.PutUvarint(enc[:], v.ExpiresAt)
|
||||
buf.Write(enc[:sz])
|
||||
buf.Write(v.Value)
|
||||
}
|
||||
|
||||
// Iterator is an interface for a basic iterator.
|
||||
type Iterator interface {
|
||||
Next()
|
||||
Rewind()
|
||||
Seek(key []byte)
|
||||
Key() []byte
|
||||
Value() ValueStruct
|
||||
Valid() bool
|
||||
|
||||
// All iterators should be closed so that file garbage collection works.
|
||||
Close() error
|
||||
}
|
68
vendor/github.com/dgraph-io/badger/y/metrics.go
generated
vendored
Normal file
68
vendor/github.com/dgraph-io/badger/y/metrics.go
generated
vendored
Normal file
|
@ -0,0 +1,68 @@
|
|||
/*
|
||||
* Copyright (C) 2017 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package y
|
||||
|
||||
import "expvar"
|
||||
|
||||
var (
|
||||
// LSMSize has size of the LSM in bytes
|
||||
LSMSize *expvar.Map
|
||||
// VlogSize has size of the value log in bytes
|
||||
VlogSize *expvar.Map
|
||||
// PendingWrites tracks the number of pending writes.
|
||||
PendingWrites *expvar.Map
|
||||
|
||||
// These are cumulative
|
||||
|
||||
// NumReads has cumulative number of reads
|
||||
NumReads *expvar.Int
|
||||
// NumWrites has cumulative number of writes
|
||||
NumWrites *expvar.Int
|
||||
// NumBytesRead has cumulative number of bytes read
|
||||
NumBytesRead *expvar.Int
|
||||
// NumBytesWritten has cumulative number of bytes written
|
||||
NumBytesWritten *expvar.Int
|
||||
// NumLSMGets is number of LMS gets
|
||||
NumLSMGets *expvar.Map
|
||||
// NumLSMBloomHits is number of LMS bloom hits
|
||||
NumLSMBloomHits *expvar.Map
|
||||
// NumGets is number of gets
|
||||
NumGets *expvar.Int
|
||||
// NumPuts is number of puts
|
||||
NumPuts *expvar.Int
|
||||
// NumBlockedPuts is number of blocked puts
|
||||
NumBlockedPuts *expvar.Int
|
||||
// NumMemtableGets is number of memtable gets
|
||||
NumMemtableGets *expvar.Int
|
||||
)
|
||||
|
||||
// These variables are global and have cumulative values for all kv stores.
|
||||
func init() {
|
||||
NumReads = expvar.NewInt("badger_disk_reads_total")
|
||||
NumWrites = expvar.NewInt("badger_disk_writes_total")
|
||||
NumBytesRead = expvar.NewInt("badger_read_bytes")
|
||||
NumBytesWritten = expvar.NewInt("badger_written_bytes")
|
||||
NumLSMGets = expvar.NewMap("badger_lsm_level_gets_total")
|
||||
NumLSMBloomHits = expvar.NewMap("badger_lsm_bloom_hits_total")
|
||||
NumGets = expvar.NewInt("badger_gets_total")
|
||||
NumPuts = expvar.NewInt("badger_puts_total")
|
||||
NumBlockedPuts = expvar.NewInt("badger_blocked_puts_total")
|
||||
NumMemtableGets = expvar.NewInt("badger_memtable_gets_total")
|
||||
LSMSize = expvar.NewMap("badger_lsm_size_bytes")
|
||||
VlogSize = expvar.NewMap("badger_vlog_size_bytes")
|
||||
PendingWrites = expvar.NewMap("badger_pending_writes_total")
|
||||
}
|
39
vendor/github.com/dgraph-io/badger/y/mmap.go
generated
vendored
Normal file
39
vendor/github.com/dgraph-io/badger/y/mmap.go
generated
vendored
Normal file
|
@ -0,0 +1,39 @@
|
|||
/*
|
||||
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package y
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// Mmap uses the mmap system call to memory-map a file. If writable is true,
|
||||
// memory protection of the pages is set so that they may be written to as well.
|
||||
func Mmap(fd *os.File, writable bool, size int64) ([]byte, error) {
|
||||
return mmap(fd, writable, size)
|
||||
}
|
||||
|
||||
// Munmap unmaps a previously mapped slice.
|
||||
func Munmap(b []byte) error {
|
||||
return munmap(b)
|
||||
}
|
||||
|
||||
// Madvise uses the madvise system call to give advise about the use of memory
|
||||
// when using a slice that is memory-mapped to a file. Set the readahead flag to
|
||||
// false if page references are expected in random order.
|
||||
func Madvise(b []byte, readahead bool) error {
|
||||
return madvise(b, readahead)
|
||||
}
|
55
vendor/github.com/dgraph-io/badger/y/mmap_darwin.go
generated
vendored
Normal file
55
vendor/github.com/dgraph-io/badger/y/mmap_darwin.go
generated
vendored
Normal file
|
@ -0,0 +1,55 @@
|
|||
/*
|
||||
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package y
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// Mmap uses the mmap system call to memory-map a file. If writable is true,
|
||||
// memory protection of the pages is set so that they may be written to as well.
|
||||
func mmap(fd *os.File, writable bool, size int64) ([]byte, error) {
|
||||
mtype := unix.PROT_READ
|
||||
if writable {
|
||||
mtype |= unix.PROT_WRITE
|
||||
}
|
||||
return unix.Mmap(int(fd.Fd()), 0, int(size), mtype, unix.MAP_SHARED)
|
||||
}
|
||||
|
||||
// Munmap unmaps a previously mapped slice.
|
||||
func munmap(b []byte) error {
|
||||
return unix.Munmap(b)
|
||||
}
|
||||
|
||||
// This is required because the unix package does not support the madvise system call on OS X.
|
||||
func madvise(b []byte, readahead bool) error {
|
||||
advice := unix.MADV_NORMAL
|
||||
if !readahead {
|
||||
advice = unix.MADV_RANDOM
|
||||
}
|
||||
|
||||
_, _, e1 := syscall.Syscall(syscall.SYS_MADVISE, uintptr(unsafe.Pointer(&b[0])),
|
||||
uintptr(len(b)), uintptr(advice))
|
||||
if e1 != 0 {
|
||||
return e1
|
||||
}
|
||||
return nil
|
||||
}
|
40
vendor/github.com/dgraph-io/badger/y/mmap_plan9.go
generated
vendored
Normal file
40
vendor/github.com/dgraph-io/badger/y/mmap_plan9.go
generated
vendored
Normal file
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
* Copyright 2020 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package y
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// Mmap uses the mmap system call to memory-map a file. If writable is true,
|
||||
// memory protection of the pages is set so that they may be written to as well.
|
||||
func mmap(fd *os.File, writable bool, size int64) ([]byte, error) {
|
||||
return nil, syscall.EPLAN9
|
||||
}
|
||||
|
||||
// Munmap unmaps a previously mapped slice.
|
||||
func munmap(b []byte) error {
|
||||
return syscall.EPLAN9
|
||||
}
|
||||
|
||||
// Madvise uses the madvise system call to give advise about the use of memory
|
||||
// when using a slice that is memory-mapped to a file. Set the readahead flag to
|
||||
// false if page references are expected in random order.
|
||||
func madvise(b []byte, readahead bool) error {
|
||||
return syscall.EPLAN9
|
||||
}
|
51
vendor/github.com/dgraph-io/badger/y/mmap_unix.go
generated
vendored
Normal file
51
vendor/github.com/dgraph-io/badger/y/mmap_unix.go
generated
vendored
Normal file
|
@ -0,0 +1,51 @@
|
|||
// +build !windows,!darwin,!plan9
|
||||
|
||||
/*
|
||||
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package y
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// Mmap uses the mmap system call to memory-map a file. If writable is true,
|
||||
// memory protection of the pages is set so that they may be written to as well.
|
||||
func mmap(fd *os.File, writable bool, size int64) ([]byte, error) {
|
||||
mtype := unix.PROT_READ
|
||||
if writable {
|
||||
mtype |= unix.PROT_WRITE
|
||||
}
|
||||
return unix.Mmap(int(fd.Fd()), 0, int(size), mtype, unix.MAP_SHARED)
|
||||
}
|
||||
|
||||
// Munmap unmaps a previously mapped slice.
|
||||
func munmap(b []byte) error {
|
||||
return unix.Munmap(b)
|
||||
}
|
||||
|
||||
// Madvise uses the madvise system call to give advise about the use of memory
|
||||
// when using a slice that is memory-mapped to a file. Set the readahead flag to
|
||||
// false if page references are expected in random order.
|
||||
func madvise(b []byte, readahead bool) error {
|
||||
flags := unix.MADV_NORMAL
|
||||
if !readahead {
|
||||
flags = unix.MADV_RANDOM
|
||||
}
|
||||
return unix.Madvise(b, flags)
|
||||
}
|
91
vendor/github.com/dgraph-io/badger/y/mmap_windows.go
generated
vendored
Normal file
91
vendor/github.com/dgraph-io/badger/y/mmap_windows.go
generated
vendored
Normal file
|
@ -0,0 +1,91 @@
|
|||
// +build windows
|
||||
|
||||
/*
|
||||
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package y
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
func mmap(fd *os.File, write bool, size int64) ([]byte, error) {
|
||||
protect := syscall.PAGE_READONLY
|
||||
access := syscall.FILE_MAP_READ
|
||||
|
||||
if write {
|
||||
protect = syscall.PAGE_READWRITE
|
||||
access = syscall.FILE_MAP_WRITE
|
||||
}
|
||||
fi, err := fd.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// In windows, we cannot mmap a file more than it's actual size.
|
||||
// So truncate the file to the size of the mmap.
|
||||
if fi.Size() < size {
|
||||
if err := fd.Truncate(size); err != nil {
|
||||
return nil, fmt.Errorf("truncate: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Open a file mapping handle.
|
||||
sizelo := uint32(size >> 32)
|
||||
sizehi := uint32(size) & 0xffffffff
|
||||
|
||||
handler, err := syscall.CreateFileMapping(syscall.Handle(fd.Fd()), nil,
|
||||
uint32(protect), sizelo, sizehi, nil)
|
||||
if err != nil {
|
||||
return nil, os.NewSyscallError("CreateFileMapping", err)
|
||||
}
|
||||
|
||||
// Create the memory map.
|
||||
addr, err := syscall.MapViewOfFile(handler, uint32(access), 0, 0, uintptr(size))
|
||||
if addr == 0 {
|
||||
return nil, os.NewSyscallError("MapViewOfFile", err)
|
||||
}
|
||||
|
||||
// Close mapping handle.
|
||||
if err := syscall.CloseHandle(syscall.Handle(handler)); err != nil {
|
||||
return nil, os.NewSyscallError("CloseHandle", err)
|
||||
}
|
||||
|
||||
// Slice memory layout
|
||||
// Copied this snippet from golang/sys package
|
||||
var sl = struct {
|
||||
addr uintptr
|
||||
len int
|
||||
cap int
|
||||
}{addr, int(size), int(size)}
|
||||
|
||||
// Use unsafe to turn sl into a []byte.
|
||||
data := *(*[]byte)(unsafe.Pointer(&sl))
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func munmap(b []byte) error {
|
||||
return syscall.UnmapViewOfFile(uintptr(unsafe.Pointer(&b[0])))
|
||||
}
|
||||
|
||||
func madvise(b []byte, readahead bool) error {
|
||||
// Do Nothing. We don’t care about this setting on Windows
|
||||
return nil
|
||||
}
|
255
vendor/github.com/dgraph-io/badger/y/watermark.go
generated
vendored
Normal file
255
vendor/github.com/dgraph-io/badger/y/watermark.go
generated
vendored
Normal file
|
@ -0,0 +1,255 @@
|
|||
/*
|
||||
* Copyright 2016-2018 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package y
|
||||
|
||||
import (
|
||||
"container/heap"
|
||||
"context"
|
||||
"sync/atomic"
|
||||
|
||||
"golang.org/x/net/trace"
|
||||
)
|
||||
|
||||
type uint64Heap []uint64
|
||||
|
||||
func (u uint64Heap) Len() int { return len(u) }
|
||||
func (u uint64Heap) Less(i, j int) bool { return u[i] < u[j] }
|
||||
func (u uint64Heap) Swap(i, j int) { u[i], u[j] = u[j], u[i] }
|
||||
func (u *uint64Heap) Push(x interface{}) { *u = append(*u, x.(uint64)) }
|
||||
func (u *uint64Heap) Pop() interface{} {
|
||||
old := *u
|
||||
n := len(old)
|
||||
x := old[n-1]
|
||||
*u = old[0 : n-1]
|
||||
return x
|
||||
}
|
||||
|
||||
// mark contains one of more indices, along with a done boolean to indicate the
|
||||
// status of the index: begin or done. It also contains waiters, who could be
|
||||
// waiting for the watermark to reach >= a certain index.
|
||||
type mark struct {
|
||||
// Either this is an (index, waiter) pair or (index, done) or (indices, done).
|
||||
index uint64
|
||||
waiter chan struct{}
|
||||
indices []uint64
|
||||
done bool // Set to true if the index is done.
|
||||
}
|
||||
|
||||
// WaterMark is used to keep track of the minimum un-finished index. Typically, an index k becomes
|
||||
// finished or "done" according to a WaterMark once Done(k) has been called
|
||||
// 1. as many times as Begin(k) has, AND
|
||||
// 2. a positive number of times.
|
||||
//
|
||||
// An index may also become "done" by calling SetDoneUntil at a time such that it is not
|
||||
// inter-mingled with Begin/Done calls.
|
||||
//
|
||||
// Since doneUntil and lastIndex addresses are passed to sync/atomic packages, we ensure that they
|
||||
// are 64-bit aligned by putting them at the beginning of the structure.
|
||||
type WaterMark struct {
|
||||
doneUntil uint64
|
||||
lastIndex uint64
|
||||
Name string
|
||||
markCh chan mark
|
||||
elog trace.EventLog
|
||||
}
|
||||
|
||||
// Init initializes a WaterMark struct. MUST be called before using it.
|
||||
func (w *WaterMark) Init(closer *Closer, eventLogging bool) {
|
||||
w.markCh = make(chan mark, 100)
|
||||
if eventLogging {
|
||||
w.elog = trace.NewEventLog("Watermark", w.Name)
|
||||
} else {
|
||||
w.elog = NoEventLog
|
||||
}
|
||||
go w.process(closer)
|
||||
}
|
||||
|
||||
// Begin sets the last index to the given value.
|
||||
func (w *WaterMark) Begin(index uint64) {
|
||||
atomic.StoreUint64(&w.lastIndex, index)
|
||||
w.markCh <- mark{index: index, done: false}
|
||||
}
|
||||
|
||||
// BeginMany works like Begin but accepts multiple indices.
|
||||
func (w *WaterMark) BeginMany(indices []uint64) {
|
||||
atomic.StoreUint64(&w.lastIndex, indices[len(indices)-1])
|
||||
w.markCh <- mark{index: 0, indices: indices, done: false}
|
||||
}
|
||||
|
||||
// Done sets a single index as done.
|
||||
func (w *WaterMark) Done(index uint64) {
|
||||
w.markCh <- mark{index: index, done: true}
|
||||
}
|
||||
|
||||
// DoneMany works like Done but accepts multiple indices.
|
||||
func (w *WaterMark) DoneMany(indices []uint64) {
|
||||
w.markCh <- mark{index: 0, indices: indices, done: true}
|
||||
}
|
||||
|
||||
// DoneUntil returns the maximum index that has the property that all indices
|
||||
// less than or equal to it are done.
|
||||
func (w *WaterMark) DoneUntil() uint64 {
|
||||
return atomic.LoadUint64(&w.doneUntil)
|
||||
}
|
||||
|
||||
// SetDoneUntil sets the maximum index that has the property that all indices
|
||||
// less than or equal to it are done.
|
||||
func (w *WaterMark) SetDoneUntil(val uint64) {
|
||||
atomic.StoreUint64(&w.doneUntil, val)
|
||||
}
|
||||
|
||||
// LastIndex returns the last index for which Begin has been called.
|
||||
func (w *WaterMark) LastIndex() uint64 {
|
||||
return atomic.LoadUint64(&w.lastIndex)
|
||||
}
|
||||
|
||||
// WaitForMark waits until the given index is marked as done.
|
||||
func (w *WaterMark) WaitForMark(ctx context.Context, index uint64) error {
|
||||
if w.DoneUntil() >= index {
|
||||
return nil
|
||||
}
|
||||
waitCh := make(chan struct{})
|
||||
w.markCh <- mark{index: index, waiter: waitCh}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-waitCh:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// process is used to process the Mark channel. This is not thread-safe,
|
||||
// so only run one goroutine for process. One is sufficient, because
|
||||
// all goroutine ops use purely memory and cpu.
|
||||
// Each index has to emit atleast one begin watermark in serial order otherwise waiters
|
||||
// can get blocked idefinitely. Example: We had an watermark at 100 and a waiter at 101,
|
||||
// if no watermark is emitted at index 101 then waiter would get stuck indefinitely as it
|
||||
// can't decide whether the task at 101 has decided not to emit watermark or it didn't get
|
||||
// scheduled yet.
|
||||
func (w *WaterMark) process(closer *Closer) {
|
||||
defer closer.Done()
|
||||
|
||||
var indices uint64Heap
|
||||
// pending maps raft proposal index to the number of pending mutations for this proposal.
|
||||
pending := make(map[uint64]int)
|
||||
waiters := make(map[uint64][]chan struct{})
|
||||
|
||||
heap.Init(&indices)
|
||||
var loop uint64
|
||||
|
||||
processOne := func(index uint64, done bool) {
|
||||
// If not already done, then set. Otherwise, don't undo a done entry.
|
||||
prev, present := pending[index]
|
||||
if !present {
|
||||
heap.Push(&indices, index)
|
||||
}
|
||||
|
||||
delta := 1
|
||||
if done {
|
||||
delta = -1
|
||||
}
|
||||
pending[index] = prev + delta
|
||||
|
||||
loop++
|
||||
if len(indices) > 0 && loop%10000 == 0 {
|
||||
min := indices[0]
|
||||
w.elog.Printf("WaterMark %s: Done entry %4d. Size: %4d Watermark: %-4d Looking for: "+
|
||||
"%-4d. Value: %d\n", w.Name, index, len(indices), w.DoneUntil(), min, pending[min])
|
||||
}
|
||||
|
||||
// Update mark by going through all indices in order; and checking if they have
|
||||
// been done. Stop at the first index, which isn't done.
|
||||
doneUntil := w.DoneUntil()
|
||||
if doneUntil > index {
|
||||
AssertTruef(false, "Name: %s doneUntil: %d. Index: %d", w.Name, doneUntil, index)
|
||||
}
|
||||
|
||||
until := doneUntil
|
||||
loops := 0
|
||||
|
||||
for len(indices) > 0 {
|
||||
min := indices[0]
|
||||
if done := pending[min]; done > 0 {
|
||||
break // len(indices) will be > 0.
|
||||
}
|
||||
// Even if done is called multiple times causing it to become
|
||||
// negative, we should still pop the index.
|
||||
heap.Pop(&indices)
|
||||
delete(pending, min)
|
||||
until = min
|
||||
loops++
|
||||
}
|
||||
|
||||
if until != doneUntil {
|
||||
AssertTrue(atomic.CompareAndSwapUint64(&w.doneUntil, doneUntil, until))
|
||||
w.elog.Printf("%s: Done until %d. Loops: %d\n", w.Name, until, loops)
|
||||
}
|
||||
|
||||
notifyAndRemove := func(idx uint64, toNotify []chan struct{}) {
|
||||
for _, ch := range toNotify {
|
||||
close(ch)
|
||||
}
|
||||
delete(waiters, idx) // Release the memory back.
|
||||
}
|
||||
|
||||
if until-doneUntil <= uint64(len(waiters)) {
|
||||
// Issue #908 showed that if doneUntil is close to 2^60, while until is zero, this loop
|
||||
// can hog up CPU just iterating over integers creating a busy-wait loop. So, only do
|
||||
// this path if until - doneUntil is less than the number of waiters.
|
||||
for idx := doneUntil + 1; idx <= until; idx++ {
|
||||
if toNotify, ok := waiters[idx]; ok {
|
||||
notifyAndRemove(idx, toNotify)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for idx, toNotify := range waiters {
|
||||
if idx <= until {
|
||||
notifyAndRemove(idx, toNotify)
|
||||
}
|
||||
}
|
||||
} // end of notifying waiters.
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-closer.HasBeenClosed():
|
||||
return
|
||||
case mark := <-w.markCh:
|
||||
if mark.waiter != nil {
|
||||
doneUntil := atomic.LoadUint64(&w.doneUntil)
|
||||
if doneUntil >= mark.index {
|
||||
close(mark.waiter)
|
||||
} else {
|
||||
ws, ok := waiters[mark.index]
|
||||
if !ok {
|
||||
waiters[mark.index] = []chan struct{}{mark.waiter}
|
||||
} else {
|
||||
waiters[mark.index] = append(ws, mark.waiter)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if mark.index > 0 {
|
||||
processOne(mark.index, mark.done)
|
||||
}
|
||||
for _, index := range mark.indices {
|
||||
processOne(index, mark.done)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
302
vendor/github.com/dgraph-io/badger/y/y.go
generated
vendored
Normal file
302
vendor/github.com/dgraph-io/badger/y/y.go
generated
vendored
Normal file
|
@ -0,0 +1,302 @@
|
|||
/*
|
||||
* Copyright 2017 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package y
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"math"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// ErrEOF indicates an end of file when trying to read from a memory mapped file
|
||||
// and encountering the end of slice.
|
||||
var ErrEOF = errors.New("End of mapped region")
|
||||
|
||||
const (
|
||||
// Sync indicates that O_DSYNC should be set on the underlying file,
|
||||
// ensuring that data writes do not return until the data is flushed
|
||||
// to disk.
|
||||
Sync = 1 << iota
|
||||
// ReadOnly opens the underlying file on a read-only basis.
|
||||
ReadOnly
|
||||
)
|
||||
|
||||
var (
|
||||
// This is O_DSYNC (datasync) on platforms that support it -- see file_unix.go
|
||||
datasyncFileFlag = 0x0
|
||||
|
||||
// CastagnoliCrcTable is a CRC32 polynomial table
|
||||
CastagnoliCrcTable = crc32.MakeTable(crc32.Castagnoli)
|
||||
|
||||
// Dummy channel for nil closers.
|
||||
dummyCloserChan = make(chan struct{})
|
||||
)
|
||||
|
||||
// OpenExistingFile opens an existing file, errors if it doesn't exist.
|
||||
func OpenExistingFile(filename string, flags uint32) (*os.File, error) {
|
||||
openFlags := os.O_RDWR
|
||||
if flags&ReadOnly != 0 {
|
||||
openFlags = os.O_RDONLY
|
||||
}
|
||||
|
||||
if flags&Sync != 0 {
|
||||
openFlags |= datasyncFileFlag
|
||||
}
|
||||
return os.OpenFile(filename, openFlags, 0)
|
||||
}
|
||||
|
||||
// CreateSyncedFile creates a new file (using O_EXCL), errors if it already existed.
|
||||
func CreateSyncedFile(filename string, sync bool) (*os.File, error) {
|
||||
flags := os.O_RDWR | os.O_CREATE | os.O_EXCL
|
||||
if sync {
|
||||
flags |= datasyncFileFlag
|
||||
}
|
||||
return os.OpenFile(filename, flags, 0600)
|
||||
}
|
||||
|
||||
// OpenSyncedFile creates the file if one doesn't exist.
|
||||
func OpenSyncedFile(filename string, sync bool) (*os.File, error) {
|
||||
flags := os.O_RDWR | os.O_CREATE
|
||||
if sync {
|
||||
flags |= datasyncFileFlag
|
||||
}
|
||||
return os.OpenFile(filename, flags, 0600)
|
||||
}
|
||||
|
||||
// OpenTruncFile opens the file with O_RDWR | O_CREATE | O_TRUNC
|
||||
func OpenTruncFile(filename string, sync bool) (*os.File, error) {
|
||||
flags := os.O_RDWR | os.O_CREATE | os.O_TRUNC
|
||||
if sync {
|
||||
flags |= datasyncFileFlag
|
||||
}
|
||||
return os.OpenFile(filename, flags, 0600)
|
||||
}
|
||||
|
||||
// SafeCopy does append(a[:0], src...).
|
||||
func SafeCopy(a, src []byte) []byte {
|
||||
return append(a[:0], src...)
|
||||
}
|
||||
|
||||
// Copy copies a byte slice and returns the copied slice.
|
||||
func Copy(a []byte) []byte {
|
||||
b := make([]byte, len(a))
|
||||
copy(b, a)
|
||||
return b
|
||||
}
|
||||
|
||||
// KeyWithTs generates a new key by appending ts to key.
|
||||
func KeyWithTs(key []byte, ts uint64) []byte {
|
||||
out := make([]byte, len(key)+8)
|
||||
copy(out, key)
|
||||
binary.BigEndian.PutUint64(out[len(key):], math.MaxUint64-ts)
|
||||
return out
|
||||
}
|
||||
|
||||
// ParseTs parses the timestamp from the key bytes.
|
||||
func ParseTs(key []byte) uint64 {
|
||||
if len(key) <= 8 {
|
||||
return 0
|
||||
}
|
||||
return math.MaxUint64 - binary.BigEndian.Uint64(key[len(key)-8:])
|
||||
}
|
||||
|
||||
// CompareKeys checks the key without timestamp and checks the timestamp if keyNoTs
|
||||
// is same.
|
||||
// a<timestamp> would be sorted higher than aa<timestamp> if we use bytes.compare
|
||||
// All keys should have timestamp.
|
||||
func CompareKeys(key1, key2 []byte) int {
|
||||
AssertTrue(len(key1) > 8 && len(key2) > 8)
|
||||
if cmp := bytes.Compare(key1[:len(key1)-8], key2[:len(key2)-8]); cmp != 0 {
|
||||
return cmp
|
||||
}
|
||||
return bytes.Compare(key1[len(key1)-8:], key2[len(key2)-8:])
|
||||
}
|
||||
|
||||
// ParseKey parses the actual key from the key bytes.
|
||||
func ParseKey(key []byte) []byte {
|
||||
if key == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
AssertTrue(len(key) > 8)
|
||||
return key[:len(key)-8]
|
||||
}
|
||||
|
||||
// SameKey checks for key equality ignoring the version timestamp suffix.
|
||||
func SameKey(src, dst []byte) bool {
|
||||
if len(src) != len(dst) {
|
||||
return false
|
||||
}
|
||||
return bytes.Equal(ParseKey(src), ParseKey(dst))
|
||||
}
|
||||
|
||||
// Slice holds a reusable buf, will reallocate if you request a larger size than ever before.
|
||||
// One problem is with n distinct sizes in random order it'll reallocate log(n) times.
|
||||
type Slice struct {
|
||||
buf []byte
|
||||
}
|
||||
|
||||
// Resize reuses the Slice's buffer (or makes a new one) and returns a slice in that buffer of
|
||||
// length sz.
|
||||
func (s *Slice) Resize(sz int) []byte {
|
||||
if cap(s.buf) < sz {
|
||||
s.buf = make([]byte, sz)
|
||||
}
|
||||
return s.buf[0:sz]
|
||||
}
|
||||
|
||||
// FixedDuration returns a string representation of the given duration with the
|
||||
// hours, minutes, and seconds.
|
||||
func FixedDuration(d time.Duration) string {
|
||||
str := fmt.Sprintf("%02ds", int(d.Seconds())%60)
|
||||
if d >= time.Minute {
|
||||
str = fmt.Sprintf("%02dm", int(d.Minutes())%60) + str
|
||||
}
|
||||
if d >= time.Hour {
|
||||
str = fmt.Sprintf("%02dh", int(d.Hours())) + str
|
||||
}
|
||||
return str
|
||||
}
|
||||
|
||||
// Closer holds the two things we need to close a goroutine and wait for it to finish: a chan
|
||||
// to tell the goroutine to shut down, and a WaitGroup with which to wait for it to finish shutting
|
||||
// down.
|
||||
type Closer struct {
|
||||
closed chan struct{}
|
||||
waiting sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewCloser constructs a new Closer, with an initial count on the WaitGroup.
|
||||
func NewCloser(initial int) *Closer {
|
||||
ret := &Closer{closed: make(chan struct{})}
|
||||
ret.waiting.Add(initial)
|
||||
return ret
|
||||
}
|
||||
|
||||
// AddRunning Add()'s delta to the WaitGroup.
|
||||
func (lc *Closer) AddRunning(delta int) {
|
||||
lc.waiting.Add(delta)
|
||||
}
|
||||
|
||||
// Signal signals the HasBeenClosed signal.
|
||||
func (lc *Closer) Signal() {
|
||||
close(lc.closed)
|
||||
}
|
||||
|
||||
// HasBeenClosed gets signaled when Signal() is called.
|
||||
func (lc *Closer) HasBeenClosed() <-chan struct{} {
|
||||
if lc == nil {
|
||||
return dummyCloserChan
|
||||
}
|
||||
return lc.closed
|
||||
}
|
||||
|
||||
// Done calls Done() on the WaitGroup.
|
||||
func (lc *Closer) Done() {
|
||||
if lc == nil {
|
||||
return
|
||||
}
|
||||
lc.waiting.Done()
|
||||
}
|
||||
|
||||
// Wait waits on the WaitGroup. (It waits for NewCloser's initial value, AddRunning, and Done
|
||||
// calls to balance out.)
|
||||
func (lc *Closer) Wait() {
|
||||
lc.waiting.Wait()
|
||||
}
|
||||
|
||||
// SignalAndWait calls Signal(), then Wait().
|
||||
func (lc *Closer) SignalAndWait() {
|
||||
lc.Signal()
|
||||
lc.Wait()
|
||||
}
|
||||
|
||||
// Throttle allows a limited number of workers to run at a time. It also
|
||||
// provides a mechanism to check for errors encountered by workers and wait for
|
||||
// them to finish.
|
||||
type Throttle struct {
|
||||
once sync.Once
|
||||
wg sync.WaitGroup
|
||||
ch chan struct{}
|
||||
errCh chan error
|
||||
finishErr error
|
||||
}
|
||||
|
||||
// NewThrottle creates a new throttle with a max number of workers.
|
||||
func NewThrottle(max int) *Throttle {
|
||||
return &Throttle{
|
||||
ch: make(chan struct{}, max),
|
||||
errCh: make(chan error, max),
|
||||
}
|
||||
}
|
||||
|
||||
// Do should be called by workers before they start working. It blocks if there
|
||||
// are already maximum number of workers working. If it detects an error from
|
||||
// previously Done workers, it would return it.
|
||||
func (t *Throttle) Do() error {
|
||||
for {
|
||||
select {
|
||||
case t.ch <- struct{}{}:
|
||||
t.wg.Add(1)
|
||||
return nil
|
||||
case err := <-t.errCh:
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Done should be called by workers when they finish working. They can also
|
||||
// pass the error status of work done.
|
||||
func (t *Throttle) Done(err error) {
|
||||
if err != nil {
|
||||
t.errCh <- err
|
||||
}
|
||||
select {
|
||||
case <-t.ch:
|
||||
default:
|
||||
panic("Throttle Do Done mismatch")
|
||||
}
|
||||
t.wg.Done()
|
||||
}
|
||||
|
||||
// Finish waits until all workers have finished working. It would return any error passed by Done.
|
||||
// If Finish is called multiple time, it will wait for workers to finish only once(first time).
|
||||
// From next calls, it will return same error as found on first call.
|
||||
func (t *Throttle) Finish() error {
|
||||
t.once.Do(func() {
|
||||
t.wg.Wait()
|
||||
close(t.ch)
|
||||
close(t.errCh)
|
||||
for err := range t.errCh {
|
||||
if err != nil {
|
||||
t.finishErr = err
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return t.finishErr
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue