feat: Waku v2 bridge

Issue #12610

vendor/github.com/status-im/status-go/db/db.go

@@ -0,0 +1,162 @@
+package db
+
+import (
+	"path/filepath"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/errors"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+type storagePrefix byte
+
+const (
+	// PeersCache is used for the db entries used for peers DB
+	PeersCache storagePrefix = iota
+	// DeduplicatorCache is used for the db entries used for messages
+	// deduplication cache
+	DeduplicatorCache
+	// MailserversCache is a list of mail servers provided by users.
+	MailserversCache
+	// TopicHistoryBucket isolated bucket for storing history metadata.
+	TopicHistoryBucket
+	// HistoryRequestBucket isolated bucket for storing list of pending requests.
+	HistoryRequestBucket
+)
+
+// NewMemoryDB returns leveldb with memory backend prefixed with a bucket.
+func NewMemoryDB() (*leveldb.DB, error) {
+	return leveldb.Open(storage.NewMemStorage(), nil)
+}
+
+// NewDBNamespace returns instance that ensures isolated operations.
+func NewDBNamespace(db Storage, prefix storagePrefix) LevelDBNamespace {
+	return LevelDBNamespace{
+		db:     db,
+		prefix: prefix,
+	}
+}
+
+// NewMemoryDBNamespace wraps in memory leveldb with provided bucket.
+// Mostly used for tests. Including tests in other packages.
+func NewMemoryDBNamespace(prefix storagePrefix) (pdb LevelDBNamespace, err error) {
+	db, err := NewMemoryDB()
+	if err != nil {
+		return pdb, err
+	}
+	return NewDBNamespace(LevelDBStorage{db: db}, prefix), nil
+}
+
+// Key creates a DB key for a specified service with specified data
+func Key(prefix storagePrefix, data ...[]byte) []byte {
+	keyLength := 1
+	for _, d := range data {
+		keyLength += len(d)
+	}
+	key := make([]byte, keyLength)
+	key[0] = byte(prefix)
+	startPos := 1
+	for _, d := range data {
+		copy(key[startPos:], d[:])
+		startPos += len(d)
+	}
+
+	return key
+}
+
+// Create returns status pointer to leveldb.DB.
+func Create(path, dbName string) (*leveldb.DB, error) {
+	// Create euphemeral storage if the node config path isn't provided
+	if path == "" {
+		return leveldb.Open(storage.NewMemStorage(), nil)
+	}
+
+	path = filepath.Join(path, dbName)
+	return Open(path, &opt.Options{OpenFilesCacheCapacity: 5})
+}
+
+// Open opens an existing leveldb database
+func Open(path string, opts *opt.Options) (db *leveldb.DB, err error) {
+	db, err = leveldb.OpenFile(path, opts)
+	if _, iscorrupted := err.(*errors.ErrCorrupted); iscorrupted {
+		log.Info("database is corrupted trying to recover", "path", path)
+		db, err = leveldb.RecoverFile(path, nil)
+	}
+	return
+}
+
+// LevelDBNamespace database where all operations will be prefixed with a certain bucket.
+type LevelDBNamespace struct {
+	db     Storage
+	prefix storagePrefix
+}
+
+func (db LevelDBNamespace) prefixedKey(key []byte) []byte {
+	endkey := make([]byte, len(key)+1)
+	endkey[0] = byte(db.prefix)
+	copy(endkey[1:], key)
+	return endkey
+}
+
+func (db LevelDBNamespace) Put(key, value []byte) error {
+	return db.db.Put(db.prefixedKey(key), value)
+}
+
+func (db LevelDBNamespace) Get(key []byte) ([]byte, error) {
+	return db.db.Get(db.prefixedKey(key))
+}
+
+// Range returns leveldb util.Range prefixed with a single byte.
+// If prefix is nil range will iterate over all records in a given bucket.
+func (db LevelDBNamespace) Range(prefix, limit []byte) *util.Range {
+	if limit == nil {
+		return util.BytesPrefix(db.prefixedKey(prefix))
+	}
+	return &util.Range{Start: db.prefixedKey(prefix), Limit: db.prefixedKey(limit)}
+}
+
+// Delete removes key from database.
+func (db LevelDBNamespace) Delete(key []byte) error {
+	return db.db.Delete(db.prefixedKey(key))
+}
+
+// NewIterator returns iterator for a given slice.
+func (db LevelDBNamespace) NewIterator(slice *util.Range) NamespaceIterator {
+	return NamespaceIterator{db.db.NewIterator(slice)}
+}
+
+// NamespaceIterator wraps leveldb iterator, works mostly the same way.
+// The only difference is that first byte of the key is dropped.
+type NamespaceIterator struct {
+	iter iterator.Iterator
+}
+
+// Key returns key of the current item.
+func (iter NamespaceIterator) Key() []byte {
+	return iter.iter.Key()[1:]
+}
+
+// Value returns actual value of the current item.
+func (iter NamespaceIterator) Value() []byte {
+	return iter.iter.Value()
+}
+
+// Error returns accumulated error.
+func (iter NamespaceIterator) Error() error {
+	return iter.iter.Error()
+}
+
+// Prev moves cursor backward.
+func (iter NamespaceIterator) Prev() bool {
+	return iter.iter.Prev()
+}
+
+// Next moves cursor forward.
+func (iter NamespaceIterator) Next() bool {
+	return iter.iter.Next()
+}

vendor/github.com/status-im/status-go/db/history.go

@@ -0,0 +1,224 @@
+package db
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb/util"
+
+	"github.com/status-im/status-go/eth-node/types"
+)
+
+var (
+	// ErrEmptyKey returned if key is not expected to be empty.
+	ErrEmptyKey = errors.New("TopicHistoryKey is empty")
+)
+
+// DB is a common interface for DB operations.
+type DB interface {
+	Get([]byte) ([]byte, error)
+	Put([]byte, []byte) error
+	Delete([]byte) error
+	Range([]byte, []byte) *util.Range
+	NewIterator(*util.Range) NamespaceIterator
+}
+
+// TopicHistoryKey defines bytes that are used as unique key for TopicHistory.
+// first 4 bytes are types.TopicType bytes
+// next 8 bytes are time.Duration encoded in big endian notation.
+type TopicHistoryKey [12]byte
+
+// LoadTopicHistoryFromKey unmarshalls key into topic and duration and loads value of topic history
+// from given database.
+func LoadTopicHistoryFromKey(db DB, key TopicHistoryKey) (th TopicHistory, err error) {
+	if (key == TopicHistoryKey{}) {
+		return th, ErrEmptyKey
+	}
+	topic := types.TopicType{}
+	copy(topic[:], key[:4])
+	duration := binary.BigEndian.Uint64(key[4:])
+	th = TopicHistory{db: db, Topic: topic, Duration: time.Duration(duration)}
+	return th, th.Load()
+}
+
+// TopicHistory stores necessary information.
+type TopicHistory struct {
+	db DB
+	// whisper topic
+	Topic types.TopicType
+
+	Duration time.Duration
+	// Timestamp that was used for the first request with this topic.
+	// Used to identify overlapping ranges.
+	First time.Time
+	// Timestamp of the last synced envelope.
+	Current time.Time
+	End     time.Time
+
+	RequestID types.Hash
+}
+
+// Key returns unique identifier for this TopicHistory.
+func (t TopicHistory) Key() TopicHistoryKey {
+	key := TopicHistoryKey{}
+	copy(key[:], t.Topic[:])
+	binary.BigEndian.PutUint64(key[4:], uint64(t.Duration))
+	return key
+}
+
+// Value marshalls TopicHistory into bytes.
+func (t TopicHistory) Value() ([]byte, error) {
+	return json.Marshal(t)
+}
+
+// Load TopicHistory from db using key and unmarshalls it.
+func (t *TopicHistory) Load() error {
+	key := t.Key()
+	if (key == TopicHistoryKey{}) {
+		return errors.New("key is empty")
+	}
+	value, err := t.db.Get(key[:])
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(value, t)
+}
+
+// Save persists TopicHistory on disk.
+func (t TopicHistory) Save() error {
+	key := t.Key()
+	val, err := t.Value()
+	if err != nil {
+		return err
+	}
+	return t.db.Put(key[:], val)
+}
+
+// Delete removes topic history from database.
+func (t TopicHistory) Delete() error {
+	key := t.Key()
+	return t.db.Delete(key[:])
+}
+
+// SameRange returns true if topic has same range, which means:
+// true if Current is zero and Duration is the same
+// and true if Current is the same
+func (t TopicHistory) SameRange(other TopicHistory) bool {
+	zero := time.Time{}
+	if t.Current == zero && other.Current == zero {
+		return t.Duration == other.Duration
+	}
+	return t.Current == other.Current
+}
+
+// Pending returns true if this topic was requested from a mail server.
+func (t TopicHistory) Pending() bool {
+	return t.RequestID != types.Hash{}
+}
+
+// HistoryRequest is kept in the database while request is in the progress.
+// Stores necessary information to identify topics with associated ranges included in the request.
+type HistoryRequest struct {
+	requestDB DB
+	topicDB   DB
+
+	histories []TopicHistory
+
+	// Generated ID
+	ID types.Hash
+	// List of the topics
+	TopicHistoryKeys []TopicHistoryKey
+}
+
+// AddHistory adds instance to internal list of instance and add instance key to the list
+// which will be persisted on disk.
+func (req *HistoryRequest) AddHistory(history TopicHistory) {
+	req.histories = append(req.histories, history)
+	req.TopicHistoryKeys = append(req.TopicHistoryKeys, history.Key())
+}
+
+// Histories returns internal lsit of topic histories.
+func (req *HistoryRequest) Histories() []TopicHistory {
+	// TODO Lazy load from database on first access
+	return req.histories
+}
+
+// Value returns content of HistoryRequest as bytes.
+func (req HistoryRequest) Value() ([]byte, error) {
+	return json.Marshal(req)
+}
+
+// Save persists all attached histories and request itself on the disk.
+func (req HistoryRequest) Save() error {
+	for i := range req.histories {
+		th := &req.histories[i]
+		th.RequestID = req.ID
+		if err := th.Save(); err != nil {
+			return err
+		}
+	}
+	val, err := req.Value()
+	if err != nil {
+		return err
+	}
+	return req.requestDB.Put(req.ID.Bytes(), val)
+}
+
+// Replace saves request with new ID and all data attached to the old one.
+func (req HistoryRequest) Replace(id types.Hash) error {
+	if (req.ID != types.Hash{}) {
+		if err := req.Delete(); err != nil {
+			return err
+		}
+	}
+	req.ID = id
+	return req.Save()
+}
+
+// Delete HistoryRequest from store and update every topic.
+func (req HistoryRequest) Delete() error {
+	return req.requestDB.Delete(req.ID.Bytes())
+}
+
+// Load reads request and topic histories content from disk and unmarshalls them.
+func (req *HistoryRequest) Load() error {
+	val, err := req.requestDB.Get(req.ID.Bytes())
+	if err != nil {
+		return err
+	}
+	return req.RawUnmarshall(val)
+}
+
+func (req *HistoryRequest) loadHistories() error {
+	for _, hk := range req.TopicHistoryKeys {
+		th, err := LoadTopicHistoryFromKey(req.topicDB, hk)
+		if err != nil {
+			return err
+		}
+		req.histories = append(req.histories, th)
+	}
+	return nil
+}
+
+// RawUnmarshall unmarshall given bytes into the structure.
+// Used in range queries to unmarshall content of the iter.Value directly into request struct.
+func (req *HistoryRequest) RawUnmarshall(val []byte) error {
+	err := json.Unmarshal(val, req)
+	if err != nil {
+		return err
+	}
+	return req.loadHistories()
+}
+
+// Includes checks if TopicHistory is included into the request.
+func (req *HistoryRequest) Includes(history TopicHistory) bool {
+	key := history.Key()
+	for i := range req.TopicHistoryKeys {
+		if key == req.TopicHistoryKeys[i] {
+			return true
+		}
+	}
+	return false
+}

vendor/github.com/status-im/status-go/db/history_store.go

@@ -0,0 +1,90 @@
+package db
+
+import (
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb/errors"
+
+	"github.com/status-im/status-go/eth-node/types"
+)
+
+// NewHistoryStore returns HistoryStore instance.
+func NewHistoryStore(storage Storage) HistoryStore {
+	return HistoryStore{
+		topicDB:   NewDBNamespace(storage, TopicHistoryBucket),
+		requestDB: NewDBNamespace(storage, HistoryRequestBucket),
+	}
+}
+
+// HistoryStore provides utility methods for quering history and requests store.
+type HistoryStore struct {
+	topicDB   DB
+	requestDB DB
+}
+
+// GetHistory creates history instance and loads history from database.
+// Returns instance populated with topic and duration if history is not found in database.
+func (h HistoryStore) GetHistory(topic types.TopicType, duration time.Duration) (TopicHistory, error) {
+	thist := h.NewHistory(topic, duration)
+	err := thist.Load()
+	if err != nil && err != errors.ErrNotFound {
+		return TopicHistory{}, err
+	}
+	return thist, nil
+}
+
+// NewRequest returns instance of the HistoryRequest.
+func (h HistoryStore) NewRequest() HistoryRequest {
+	return HistoryRequest{requestDB: h.requestDB, topicDB: h.topicDB}
+}
+
+// NewHistory creates TopicHistory object with required values.
+func (h HistoryStore) NewHistory(topic types.TopicType, duration time.Duration) TopicHistory {
+	return TopicHistory{db: h.topicDB, Duration: duration, Topic: topic}
+}
+
+// GetRequest loads HistoryRequest from database.
+func (h HistoryStore) GetRequest(id types.Hash) (HistoryRequest, error) {
+	req := HistoryRequest{requestDB: h.requestDB, topicDB: h.topicDB, ID: id}
+	err := req.Load()
+	if err != nil {
+		return HistoryRequest{}, err
+	}
+	return req, nil
+}
+
+// GetAllRequests loads all not-finished history requests from database.
+func (h HistoryStore) GetAllRequests() ([]HistoryRequest, error) {
+	rst := []HistoryRequest{}
+	iter := h.requestDB.NewIterator(h.requestDB.Range(nil, nil))
+	for iter.Next() {
+		req := HistoryRequest{
+			requestDB: h.requestDB,
+			topicDB:   h.topicDB,
+		}
+		err := req.RawUnmarshall(iter.Value())
+		if err != nil {
+			return nil, err
+		}
+		rst = append(rst, req)
+	}
+	return rst, nil
+}
+
+// GetHistoriesByTopic returns all histories with a given topic.
+// This is needed when we will have multiple range per single topic.
+// TODO explain
+func (h HistoryStore) GetHistoriesByTopic(topic types.TopicType) ([]TopicHistory, error) {
+	rst := []TopicHistory{}
+	iter := h.topicDB.NewIterator(h.topicDB.Range(topic[:], nil))
+	for iter.Next() {
+		key := TopicHistoryKey{}
+		copy(key[:], iter.Key())
+		th, err := LoadTopicHistoryFromKey(h.topicDB, key)
+		if err != nil {
+			return nil, err
+		}
+		rst = append(rst, th)
+	}
+	return rst, nil
+}

vendor/github.com/status-im/status-go/db/storage.go

@@ -0,0 +1,75 @@
+package db
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// Storage is an interface for common db operations.
+type Storage interface {
+	Put([]byte, []byte) error
+	Delete([]byte) error
+	Get([]byte) ([]byte, error)
+	NewIterator(*util.Range) iterator.Iterator
+}
+
+// CommitStorage allows to write all tx/batched values atomically.
+type CommitStorage interface {
+	Storage
+	Commit() error
+}
+
+// TransactionalStorage adds transaction features on top of regular storage.
+type TransactionalStorage interface {
+	Storage
+	NewTx() CommitStorage
+}
+
+// NewMemoryLevelDBStorage returns LevelDBStorage instance with in memory leveldb backend.
+func NewMemoryLevelDBStorage() (LevelDBStorage, error) {
+	mdb, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		return LevelDBStorage{}, err
+	}
+	return NewLevelDBStorage(mdb), nil
+}
+
+// NewLevelDBStorage creates new LevelDBStorage instance.
+func NewLevelDBStorage(db *leveldb.DB) LevelDBStorage {
+	return LevelDBStorage{db: db}
+}
+
+// LevelDBStorage wrapper around leveldb.DB.
+type LevelDBStorage struct {
+	db *leveldb.DB
+}
+
+// Put upserts given key/value pair.
+func (db LevelDBStorage) Put(key, buf []byte) error {
+	return db.db.Put(key, buf, nil)
+}
+
+// Delete removes given key from database..
+func (db LevelDBStorage) Delete(key []byte) error {
+	return db.db.Delete(key, nil)
+}
+
+// Get returns value for a given key.
+func (db LevelDBStorage) Get(key []byte) ([]byte, error) {
+	return db.db.Get(key, nil)
+}
+
+// NewIterator returns new leveldb iterator.Iterator instance for a given range.
+func (db LevelDBStorage) NewIterator(slice *util.Range) iterator.Iterator {
+	return db.db.NewIterator(slice, nil)
+}
+
+// NewTx is a wrapper around leveldb.Batch that allows to write atomically.
+func (db LevelDBStorage) NewTx() CommitStorage {
+	return LevelDBTx{
+		batch: &leveldb.Batch{},
+		db:    db,
+	}
+}

vendor/github.com/status-im/status-go/db/tx.go

@@ -0,0 +1,40 @@
+package db
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// LevelDBTx doesn't provide any read isolation. It allows committing all writes atomically (put/delete).
+type LevelDBTx struct {
+	batch *leveldb.Batch
+	db    LevelDBStorage
+}
+
+// Put adds key/value to associated batch.
+func (tx LevelDBTx) Put(key, buf []byte) error {
+	tx.batch.Put(key, buf)
+	return nil
+}
+
+// Delete adds delete operation to associated batch.
+func (tx LevelDBTx) Delete(key []byte) error {
+	tx.batch.Delete(key)
+	return nil
+}
+
+// Get reads from currently committed state.
+func (tx LevelDBTx) Get(key []byte) ([]byte, error) {
+	return tx.db.Get(key)
+}
+
+// NewIterator returns iterator.Iterator that will read from currently committed state.
+func (tx LevelDBTx) NewIterator(slice *util.Range) iterator.Iterator {
+	return tx.db.NewIterator(slice)
+}
+
+// Commit writes batch atomically.
+func (tx LevelDBTx) Commit() error {
+	return tx.db.db.Write(tx.batch, nil)
+}