Add leveldb to db backend and fix error wrapping

go.mod

@@ -16,11 +16,14 @@ require (
 	github.com/go-redis/redis/v8 v8.3.0
 	github.com/gomodule/redigo v1.8.2 // indirect
 	github.com/imdario/mergo v0.3.9
+	github.com/pkg/errors v0.9.1
 	github.com/pkg/profile v1.4.0
 	github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46
 	github.com/smartystreets/goconvey v1.6.4
+	github.com/syndtr/goleveldb v1.0.0
 	github.com/urfave/cli v1.22.3
 	github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb // indirect
 	golang.org/x/sys v0.0.0-20200519105757-fe76b779f299
+	google.golang.org/protobuf v1.23.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 )

go.sum

@@ -74,6 +74,7 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k=
@@ -114,9 +115,11 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4=
 github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs=
@@ -124,6 +127,8 @@ github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/profile v1.4.0 h1:uCmaf4vVbWAOZz36k1hrQD7ijGRzLwaME8Am/7a4jZI=
 github.com/pkg/profile v1.4.0/go.mod h1:NWz/XGvpEW1FyYQ7fCx4dqYBLlfTcE+A9FLAkNKqjFE=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -157,6 +162,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
+github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
@@ -217,6 +224,7 @@ gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

manager/db.go

@@ -9,6 +9,8 @@ import (
 	"github.com/boltdb/bolt"
 	"github.com/dgraph-io/badger/v2"
 	"github.com/go-redis/redis/v8"
+	"github.com/pkg/errors"
+	"github.com/syndtr/goleveldb/leveldb"
 
 	. "github.com/tuna/tunasync/internal"
 )
@@ -86,6 +88,19 @@ func makeDBAdapter(dbType string, dbFile string) (dbAdapter, error) {
 		}
 		err = kv.Init()
 		return &kv, err
+	} else if dbType == "leveldb" {
+		innerDB, err := leveldb.OpenFile(dbFile, nil)
+		if err != nil {
+			return nil, err
+		}
+		db := leveldbAdapter{
+			db: innerDB,
+		}
+		kv := kvDBAdapter{
+			db: &db,
+		}
+		err = kv.Init()
+		return &kv, err
 	}
 	// unsupported db-type
 	return nil, fmt.Errorf("unsupported db-type: %s", dbType)
@@ -116,7 +131,7 @@ func (b *kvDBAdapter) ListWorkers() (ws []WorkerStatus, err error) {
 	for _, v := range workers {
 		jsonErr := json.Unmarshal(v, &w)
 		if jsonErr != nil {
-			err = fmt.Errorf("%s; %s", err.Error(), jsonErr)
+			err = errors.Wrap(err, jsonErr.Error())
 			continue
 		}
 		ws = append(ws, w)
@@ -193,7 +208,7 @@ func (b *kvDBAdapter) ListMirrorStatus(workerID string) (ms []MirrorStatus, err
 			var m MirrorStatus
 			jsonErr := json.Unmarshal(v, &m)
 			if jsonErr != nil {
-				err = fmt.Errorf("%s; %s", err.Error(), jsonErr)
+				err = errors.Wrap(err, jsonErr.Error())
 				continue
 			}
 			ms = append(ms, m)
@@ -213,7 +228,7 @@ func (b *kvDBAdapter) ListAllMirrorStatus() (ms []MirrorStatus, err error) {
 		var m MirrorStatus
 		jsonErr := json.Unmarshal(v, &m)
 		if jsonErr != nil {
-			err = fmt.Errorf("%s; %s", err.Error(), jsonErr)
+			err = errors.Wrap(err, jsonErr.Error())
 			continue
 		}
 		ms = append(ms, m)
@@ -232,13 +247,13 @@ func (b *kvDBAdapter) FlushDisabledJobs() (err error) {
 		var m MirrorStatus
 		jsonErr := json.Unmarshal(v, &m)
 		if jsonErr != nil {
-			err = fmt.Errorf("%s; %s", err.Error(), jsonErr)
+			err = errors.Wrap(err, jsonErr.Error())
 			continue
 		}
 		if m.Status == Disabled || len(m.Name) == 0 {
 			deleteErr := b.db.Delete(_statusBucketKey, k)
 			if deleteErr != nil {
-				err = fmt.Errorf("%s; %s", err.Error(), deleteErr)
+				err = errors.Wrap(err, deleteErr.Error())
 			}
 		}
 	}

manager/db_leveldb.go

@@ -0,0 +1,51 @@
+package manager
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// implement kv interface backed by leveldb
+type leveldbAdapter struct {
+	db *leveldb.DB
+}
+
+func (b *leveldbAdapter) InitBucket(bucket string) (err error) {
+	// no-op
+	return
+}
+
+func (b *leveldbAdapter) Get(bucket string, key string) (v []byte, err error) {
+	v, err = b.db.Get([]byte(bucket+key), nil)
+	return
+}
+
+func (b *leveldbAdapter) GetAll(bucket string) (m map[string][]byte, err error) {
+	it := b.db.NewIterator(util.BytesPrefix([]byte(bucket)), nil)
+	defer it.Release()
+	m = make(map[string][]byte)
+	for it.Next() {
+		k := string(it.Key())
+		actualKey := k[len(bucket):]
+		// it.Value() changes on next iteration
+		val := it.Value()
+		v := make([]byte, len(val))
+		copy(v, it.Value())
+		m[actualKey] = v
+	}
+	return
+}
+
+func (b *leveldbAdapter) Put(bucket string, key string, value []byte) error {
+	err := b.db.Put([]byte(bucket+key), []byte(value), nil)
+	return err
+}
+
+func (b *leveldbAdapter) Delete(bucket string, key string) error {
+	err := b.db.Delete([]byte(bucket+key), nil)
+	return err
+}
+
+func (b *leveldbAdapter) Close() error {
+	return b.db.Close()
+}

manager/db_test.go

@@ -216,4 +216,22 @@ func TestDBAdapter(t *testing.T) {
 
 		DBAdapterTest(badgerDB)
 	})
+
+	Convey("leveldbAdapter should work", t, func() {
+		tmpDir, err := ioutil.TempDir("", "tunasync")
+		defer os.RemoveAll(tmpDir)
+		So(err, ShouldBeNil)
+
+		dbType, dbFile := "leveldb", filepath.Join(tmpDir, "leveldb.db")
+		leveldbDB, err := makeDBAdapter(dbType, dbFile)
+		So(err, ShouldBeNil)
+
+		defer func() {
+			// close leveldbDB
+			err := leveldbDB.Close()
+			So(err, ShouldBeNil)
+		}()
+
+		DBAdapterTest(leveldbDB)
+	})
 }