go_dreamfactory/sys/db/dbconn.go

package db

import (
	"context"
	"fmt"
	"go_dreamfactory/comm"
	"go_dreamfactory/lego/core"
	"go_dreamfactory/lego/sys/log"
	"go_dreamfactory/lego/sys/mgo"
	lgredis "go_dreamfactory/lego/sys/redis"
	"go_dreamfactory/lego/utils/codec"
	"go_dreamfactory/lego/utils/codec/codecore"
	"go_dreamfactory/lego/utils/codec/json"
	"reflect"
	"unsafe"

	"github.com/go-redis/redis/v8"

	"github.com/modern-go/reflect2"
	"go.mongodb.org/mongo-driver/bson"
	"go.mongodb.org/mongo-driver/bson/primitive"
	"go.mongodb.org/mongo-driver/mongo"
)

var defconf = &codecore.Config{
	SortMapKeys:           true,
	IndentionStep:         1,
	OnlyTaggedField:       false,
	DisallowUnknownFields: false,
	CaseSensitive:         false,
	TagKey:                "json",
}

const (
	DB_ModelTable core.SqlTable = "model_log"
)

func newDBConn(conf DBConfig) (conn *DBConn, err error) {
	conn = &DBConn{}
	if conf.RedisIsCluster {
		conn.Redis, err = lgredis.NewSys(
			lgredis.SetRedisType(lgredis.Redis_Cluster),
			lgredis.SetRedis_Cluster_Addr(conf.RedisAddr),
			lgredis.SetRedis_Cluster_Password(conf.RedisPassword))
	} else {
		conn.Redis, err = lgredis.NewSys(
			lgredis.SetRedisType(lgredis.Redis_Single),
			lgredis.SetRedis_Single_Addr(conf.RedisAddr[0]),
			lgredis.SetRedis_Single_Password(conf.RedisPassword),
			lgredis.SetRedis_Single_DB(conf.RedisDB),
		)
	}
	if err != nil {
		log.Error(err.Error(), log.Field{"config", conf})
		return
	}
	if conn.Mgo, err = mgo.NewSys(
		mgo.SetMongodbUrl(conf.MongodbUrl),
		mgo.SetMongodbDatabase(conf.MongodbDatabase),
	); err != nil {
		log.Error(err.Error(), log.Field{"config", conf})
		return
	}
	return
}

type DBConn struct {
	Redis lgredis.ISys
	Mgo   mgo.ISys
}

func NewDBModel(TableName string, conn *DBConn) *DBModel {
	return &DBModel{
		TableName: TableName,
		Redis:     conn.Redis,
		DB:        conn.Mgo,
	}
}

//DB模型
type DBModel struct {
	TableName string
	Redis     lgredis.ISys
	DB        mgo.ISys
}

func (this *DBModel) ukey(uid string) string {
	return fmt.Sprintf("%s:%s{%s}", this.TableName, uid, this.TableName)
}
func (this *DBModel) ukeylist(uid string, id string) string {
	return fmt.Sprintf("%s:%s-%s{%s}", this.TableName, uid, id, this.TableName)
}
func (this *DBModel) InsertModelLogs(table string, uID string, target interface{}) (err error) {

	data := &comm.Autogenerated{
		ID:  primitive.NewObjectID().Hex(),
		UID: uID,
		Act: string(comm.LogHandleType_Insert),
	}
	data.D = append(data.D, table)  // D[0]
	data.D = append(data.D, target) // D[1]

	_, err = this.DB.InsertOne(DB_ModelTable, data)
	if err != nil {
		log.Errorf("insert model db err %v", err)
	}

	return err
}
func (this *DBModel) DeleteModelLogs(table string, uID string, where interface{}) (err error) {

	data := &comm.Autogenerated{
		ID:  primitive.NewObjectID().Hex(),
		UID: uID,
		Act: string(comm.LogHandleType_Delete),
	}

	data.D = append(data.D, table) // D[0]
	data.D = append(data.D, where) // D[1]

	_, err = this.DB.InsertOne(DB_ModelTable, data)
	if err != nil {
		log.Errorf("insert model db err %v", err)
	}

	return err
}

func (this *DBModel) UpdateModelLogs(table string, uID string, where bson.M, target interface{}) (err error) {

	data := &comm.Autogenerated{
		ID:  primitive.NewObjectID().Hex(),
		UID: uID,
		Act: string(comm.LogHandleType_Update),
	}
	data.D = append(data.D, table)  // D[0]
	data.D = append(data.D, where)  // D[1]
	data.D = append(data.D, target) // D[2]

	_, err = this.DB.InsertOne(DB_ModelTable, data)
	if err != nil {
		log.Errorf("insert model db err %v", err)
	}

	return err
}

//添加新的数据
func (this *DBModel) Add(uid string, data interface{}, opt ...DBOption) (err error) {
	if err = this.Redis.HMSet(this.ukey(uid), data); err != nil {
		return
	}
	option := newDBOption(opt...)
	if option.IsMgoLog {
		err = this.InsertModelLogs(this.TableName, uid, []interface{}{data})
	}
	if option.Expire > 0 {
		err = this.Redis.Expire(this.ukey(uid), option.Expire)
	}
	return
}

//添加新的数据到列表
func (this *DBModel) AddList(uid string, id string, data interface{}, opt ...DBOption) (err error) {
	key := this.ukeylist(uid, id)
	if err = this.Redis.HMSet(key, data); err != nil {
		return
	}
	if err = this.Redis.HSet(this.ukey(uid), id, key); err != nil {
		return
	}
	option := newDBOption(opt...)
	if option.IsMgoLog {
		err = this.InsertModelLogs(this.TableName, uid, []interface{}{data})
	}
	if option.Expire > 0 {
		err = this.Redis.Expire(this.ukey(uid), option.Expire)
	}
	return
}

//添加新的多个数据到列表 data map[string]type
func (this *DBModel) AddLists(uid string, data interface{}, opt ...DBOption) (err error) {
	vof := reflect.ValueOf(data)
	if !vof.IsValid() {
		return fmt.Errorf("Model_Comp: AddLists(nil)")
	}
	if vof.Kind() != reflect.Map {
		return fmt.Errorf("Model_Comp: AddLists(non-pointer %T)", data)
	}
	listskeys := make(map[string]string)
	keys := vof.MapKeys()
	lists := make([]interface{}, len(keys))
	pipe := this.Redis.RedisPipe(context.TODO())
	for _, k := range keys {
		value := vof.MapIndex(k)
		keydata := k.Interface().(string)
		valuedata := value.Interface()
		key := this.ukeylist(uid, keydata)
		pipe.HMSet(key, valuedata)
		listskeys[keydata] = key
	}
	pipe.HMSetForMap(this.ukey(uid), listskeys)
	if _, err = pipe.Exec(); err != nil {
		return
	}

	option := newDBOption(opt...)
	if option.IsMgoLog {
		err = this.InsertModelLogs(this.TableName, uid, lists)
	}
	if option.Expire > 0 {
		this.Redis.Expire(this.ukey(uid), option.Expire)
		for _, v := range listskeys {
			this.Redis.Expire(v, option.Expire)
		}
	}
	return
}

//添加队列
func (this *DBModel) AddQueues(key string, uplimit int64, data interface{}) (outkey []string, err error) {
	vof := reflect.ValueOf(data)
	if !vof.IsValid() {
		err = fmt.Errorf("Model_Comp: AddLists(nil)")
		return
	}
	if vof.Kind() != reflect.Map {
		err = fmt.Errorf("Model_Comp: AddLists(non-pointer %T)", data)
		return
	}
	keys := make([]string, 0)
	pipe := this.Redis.RedisPipe(context.TODO())
	for _, k := range vof.MapKeys() {
		value := vof.MapIndex(k)
		tkey := k.Interface().(string)
		valuedata := value.Interface()
		pipe.HMSet(tkey, valuedata)
		keys = append(keys, tkey)
	}
	pipe.RPushForStringSlice(key, keys...)
	lcmd := pipe.Llen(key)

	if _, err = pipe.Exec(); err == nil {
		if lcmd.Val() > uplimit*3 { //操作3倍上限移除多余数据
			off := uplimit - lcmd.Val()
			if outkey, err = this.Redis.LRangeToStringSlice(key, 0, int(off-1)).Result(); err != nil {
				return
			}
			pipe.Ltrim(key, int(off), -1)
			for _, v := range outkey {
				pipe.Delete(v)
			}
			_, err = pipe.Exec()
		}
	}
	return
}

//修改数据多个字段 uid 作为主键
func (this *DBModel) Change(uid string, data map[string]interface{}, opt ...DBOption) (err error) {
	if err = this.Redis.HMSet(this.ukey(uid), data); err != nil {
		return
	}
	option := newDBOption(opt...)
	if option.IsMgoLog {
		err = this.UpdateModelLogs(this.TableName, uid, bson.M{"uid": uid}, data)
	}
	if option.Expire > 0 {
		this.Redis.Expire(this.ukey(uid), option.Expire)
	}
	return nil
}

//修改数据多个字段 uid 作为主键
func (this *DBModel) ChangeList(uid string, _id string, data map[string]interface{}, opt ...DBOption) (err error) {
	if err = this.Redis.HMSet(this.ukeylist(uid, _id), data); err != nil {
		return
	}

	option := newDBOption(opt...)
	if option.IsMgoLog {
		err = this.UpdateModelLogs(this.TableName, uid, bson.M{"_id": _id, "uid": uid}, data)
	}
	if option.Expire > 0 {
		this.Redis.Expire(this.ukey(uid), option.Expire)
	}
	return nil
}

//读取全部数据
func (this *DBModel) Get(uid string, data interface{}, opt ...DBOption) (err error) {
	if err = this.Redis.HGetAll(this.ukey(uid), data); err != nil && err != lgredis.RedisNil {
		return
	}
	if err == lgredis.RedisNil {
		if err = this.DB.FindOne(core.SqlTable(this.TableName), bson.M{"uid": uid}).Decode(data); err != nil {
			return
		}
		err = this.Redis.HMSet(this.ukey(uid), data)
	}
	option := newDBOption(opt...)
	if option.Expire > 0 {
		this.Redis.Expire(this.ukey(uid), option.Expire)
	}
	return
}

//获取列表数据 注意 data 必须是 切片的指针 *[]type
func (this *DBModel) GetList(uid string, data interface{}) (err error) {
	var (
		dtype         reflect2.Type
		dkind         reflect.Kind
		sType         reflect2.Type
		sliceType     *reflect2.UnsafeSliceType
		sliceelemType reflect2.Type
		decoder       codecore.IDecoderMapJson
		encoder       codecore.IEncoderMapJson
		dptr          unsafe.Pointer
		elemPtr       unsafe.Pointer
		n             int
		ok            bool
		keys          map[string]string
		tempdata      map[string]string
		result        []*redis.StringStringMapCmd
		c             *mongo.Cursor
	)
	keys = make(map[string]string)
	dptr = reflect2.PtrOf(data)
	dtype = reflect2.TypeOf(data)
	dkind = dtype.Kind()
	if dkind != reflect.Ptr {
		err = fmt.Errorf("MCompModel: GetList(non-pointer %T)", data)
		return
	}
	sType = dtype.(*reflect2.UnsafePtrType).Elem()
	if sType.Kind() != reflect.Slice {
		err = fmt.Errorf("MCompModel: GetList(data no slice %T)", data)
		return
	}
	sliceType = sType.(*reflect2.UnsafeSliceType)
	sliceelemType = sliceType.Elem()
	if sliceelemType.Kind() != reflect.Ptr {
		err = fmt.Errorf("MCompModel: GetList(sliceelemType non-pointer %T)", data)
		return
	}
	if decoder, ok = codec.DecoderOf(sliceelemType, defconf).(codecore.IDecoderMapJson); !ok {
		err = fmt.Errorf("MCompModel: GetList(data not support MarshalMapJson %T)", data)
		return
	}
	sliceelemType = sliceelemType.(*reflect2.UnsafePtrType).Elem()
	pipe := this.Redis.RedisPipe(context.TODO())
	if keys, err = this.Redis.HGetAllToMapString(this.ukey(uid)); err == nil {
		result = make([]*redis.StringStringMapCmd, 0)
		for _, v := range keys {
			cmd := pipe.HGetAllToMapString(v)
			result = append(result, cmd)
		}
		pipe.Exec()
		for _, v := range result {
			tempdata, err = v.Result()
			sliceType.UnsafeGrow(dptr, n+1)
			elemPtr = sliceType.UnsafeGetIndex(dptr, n)
			if *((*unsafe.Pointer)(elemPtr)) == nil {
				newPtr := sliceelemType.UnsafeNew()
				if err = decoder.DecodeForMapJson(newPtr, json.GetReader([]byte{}), tempdata); err != nil {
					log.Errorf("err:%v", err)
					return
				}
				*((*unsafe.Pointer)(elemPtr)) = newPtr
			} else {
				decoder.DecodeForMapJson(*((*unsafe.Pointer)(elemPtr)), json.GetReader([]byte{}), tempdata)
			}
			n++
		}
	}
	if err == lgredis.RedisNil {
		//query from mgo
		if c, err = this.DB.Find(core.SqlTable(this.TableName), bson.M{"uid": uid}); err != nil {
			return err
		} else {
			if encoder, ok = codec.EncoderOf(sliceelemType, defconf).(codecore.IEncoderMapJson); !ok {
				err = fmt.Errorf("MCompModel: GetList(data not support UnMarshalMapJson %T)", data)
				return
			}
			n = 0
			for c.Next(context.Background()) {
				_id := c.Current.Lookup("_id").StringValue()
				sliceType.UnsafeGrow(dptr, n+1)
				elemPtr = sliceType.UnsafeGetIndex(dptr, n)
				if *((*unsafe.Pointer)(elemPtr)) == nil {
					newPtr := sliceelemType.UnsafeNew()
					*((*unsafe.Pointer)(elemPtr)) = newPtr
				}
				elem := sliceType.GetIndex(data, n)
				if err = c.Decode(elem); err != nil {
					return
				}
				if tempdata, err = encoder.EncodeToMapJson(*((*unsafe.Pointer)(elemPtr)), json.GetWriter()); err != nil {
					return
				}
				key := this.ukeylist(uid, _id)
				pipe.HMSetForMap(key, tempdata)
				keys[_id] = key
				n++
			}
			if len(keys) > 0 {
				pipe.HMSetForMap(this.ukey(uid), keys)
				_, err = pipe.Exec()
			}
		}
	}

	return err
}

//获取队列数据
func (this *DBModel) GetQueues(key string, count int, data interface{}) (err error) {
	var (
		dtype         reflect2.Type
		dkind         reflect.Kind
		sType         reflect2.Type
		sliceType     *reflect2.UnsafeSliceType
		sliceelemType reflect2.Type
		dptr          unsafe.Pointer
		elemPtr       unsafe.Pointer
		decoder       codecore.IDecoderMapJson
		ok            bool
		n             int
		keys          = make([]string, 0)
		result        = make([]*redis.StringStringMapCmd, 0)
		tempdata      map[string]string
	)

	dptr = reflect2.PtrOf(data)
	dtype = reflect2.TypeOf(data)
	dkind = dtype.Kind()
	if dkind != reflect.Ptr {
		err = fmt.Errorf("MCompModel: GetList(non-pointer %T)", data)
		return
	}
	sType = dtype.(*reflect2.UnsafePtrType).Elem()
	if sType.Kind() != reflect.Slice {
		err = fmt.Errorf("MCompModel: GetList(data no slice %T)", data)
		return
	}
	sliceType = sType.(*reflect2.UnsafeSliceType)
	sliceelemType = sliceType.Elem()
	if sliceelemType.Kind() != reflect.Ptr {
		err = fmt.Errorf("MCompModel: GetList(sliceelemType non-pointer %T)", data)
		return
	}
	if decoder, ok = codec.DecoderOf(sliceelemType, defconf).(codecore.IDecoderMapJson); !ok {
		err = fmt.Errorf("MCompModel: GetList(data not support MarshalMapJson %T)", data)
		return
	}
	sliceelemType = sliceelemType.(*reflect2.UnsafePtrType).Elem()

	pipe := this.Redis.RedisPipe(context.TODO())
	if keys, err = this.Redis.LRangeToStringSlice(key, -1*count, -1).Result(); err == nil {
		result = make([]*redis.StringStringMapCmd, 0)
		for _, v := range keys {
			cmd := pipe.HGetAllToMapString(v)
			result = append(result, cmd)
		}
		pipe.Exec()
		for _, v := range result {
			tempdata, err = v.Result()
			sliceType.UnsafeGrow(dptr, n+1)
			elemPtr = sliceType.UnsafeGetIndex(dptr, n)
			if *((*unsafe.Pointer)(elemPtr)) == nil {
				newPtr := sliceelemType.UnsafeNew()
				if err = decoder.DecodeForMapJson(newPtr, json.GetReader([]byte{}), tempdata); err != nil {
					log.Errorf("err:%v", err)
					return
				}
				*((*unsafe.Pointer)(elemPtr)) = newPtr
			} else {
				decoder.DecodeForMapJson(*((*unsafe.Pointer)(elemPtr)), json.GetReader([]byte{}), tempdata)
			}
			n++
		}
	}
	return
}

//读取单个数据中 多个字段数据
func (this *DBModel) GetFields(uid string, data interface{}, fields ...string) (err error) {
	this.Redis.HMGet(this.ukey(uid), data, fields...)
	return
}

//读取List列表中单个数据中 多个字段数据
func (this *DBModel) GetListFields(uid string, id string, data interface{}, fields ...string) (err error) {
	this.Redis.HMGet(this.ukeylist(uid, id), data, fields...)
	return
}

//读取列表数据中单个数据
func (this *DBModel) GetListObj(uid string, id string, data interface{}) (err error) {
	err = this.Redis.HGetAll(this.ukeylist(uid, id), data)
	return
}

//删除用户数据
func (this *DBModel) Del(uid string, opt ...DBOption) (err error) {
	err = this.Redis.Delete(this.ukey(uid))
	if err != nil {
		return err
	}
	option := newDBOption(opt...)
	if option.IsMgoLog {
		err = this.DeleteModelLogs(this.TableName, uid, bson.M{"uid": uid})
	}
	return nil
}

//删除多条数据
func (this *DBModel) DelListlds(uid string, ids ...string) (err error) {
	listkey := this.ukey(uid)
	pipe := this.Redis.RedisPipe(context.TODO())
	for _, v := range ids {
		key := this.ukeylist(uid, v)
		pipe.Delete(key)
	}
	pipe.HDel(listkey, ids...)
	if _, err = pipe.Exec(); err == nil {
		err = this.DeleteModelLogs(this.TableName, uid, bson.M{"_id": bson.M{"$in": ids}})
	}
	return
}

//批量删除数据
func (this *DBModel) BatchDelLists(uid string) (err error) {
	var keys map[string]string
	pipe := this.Redis.RedisPipe(context.TODO())
	if keys, err = this.Redis.HGetAllToMapString(this.ukey(uid)); err == nil {
		for _, v := range keys {
			pipe.Delete(v)
		}
		pipe.Delete(this.ukey(uid))
		pipe.Exec()
	}
	return
}