1.人事初始版

.gitignore

@@ -0,0 +1,132 @@
+# ---> Python
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+.idea

Pipfile

@@ -0,0 +1,29 @@
+[[source]]
+url = "https://pypi.douban.com/simple"
+verify_ssl = false
+name = "pypi"
+
+[packages]
+fastapi = "*"
+sqlalchemy = "*"
+pymongo = "*"
+uvicorn = "*"
+motor = "*"
+python-jose = "*"
+passlib = "*"
+pydantic = {extras = ["email"], version = "*"}
+emails = "*"
+python-multipart = "*"
+gunicorn = "*"
+simpleeval = "*"
+aredis = "*"
+aioch = "*"
+aioredis = "*"
+redis = "*"
+bcrypt = "*"
+pandas = "==1.2.3"
+
+[dev-packages]
+
+[requires]
+python_version = "3.8"

README.md

@@ -0,0 +1,7 @@
+
+
+人事后端  使用 mongo 数据库
+简历数据 使用CK数据库
+
+
+

api/api_v1/api.py

@@ -0,0 +1,40 @@
+from fastapi import APIRouter
+from api.api_v1.endpoints import user
+from .endpoints import project
+from .endpoints import folder
+from .endpoints import space
+from .endpoints import dashboard
+from .endpoints import report
+# from .endpoints import authority
+from .endpoints import data_mana
+from .endpoints import query
+from .endpoints import xquery
+from .endpoints import data_auth
+from .endpoints import event_mana
+from .endpoints import test
+from .authz import authz
+from .check_data import controller as check_data
+from .user_label import controller as user_label
+
+api_router = APIRouter()
+api_router.include_router(test.router, tags=["test"], prefix='/test')
+
+api_router.include_router(user.router, tags=["用户接口"], prefix='/user')
+api_router.include_router(project.router, tags=["项目接口"], prefix='/project')
+api_router.include_router(folder.router, tags=["文件夹接口"], prefix='/folder')
+api_router.include_router(space.router, tags=["空间接口"], prefix='/space')
+api_router.include_router(dashboard.router, tags=["看板接口"], prefix='/dashboard')
+api_router.include_router(report.router, tags=["报表接口"], prefix='/report')
+
+# api_router.include_router(authority.router, tags=["权限管理接口"], prefix='/authority')
+api_router.include_router(data_auth.router, tags=["数据权限"], prefix='/data_auth')
+
+api_router.include_router(data_mana.router, tags=["数据管理"], prefix='/data_mana')
+api_router.include_router(event_mana.router, tags=["数据管理"], prefix='/data_mana')
+
+api_router.include_router(query.router, tags=["ck"], prefix='/ck')
+api_router.include_router(xquery.router, tags=["xck"], prefix='/ck')
+
+api_router.include_router(authz.router, tags=["api接口管理"], prefix='/authz')
+api_router.include_router(check_data.router, tags=["打点验证"], prefix='/check_data')
+api_router.include_router(user_label.router, tags=["用户标签"], prefix='/user_label')

api/api_v1/authz/authz.py

@@ -0,0 +1,616 @@
+from typing import Any
+
+from fastapi import APIRouter, Depends, Request
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import crud
+import schemas
+from api import deps
+from db import get_database
+from db.ckdb import CKDrive, get_ck_db
+from db.redisdb import RedisDrive, get_redis_pool
+from models.behavior_analysis import BehaviorAnalysis
+from utils import casbin_enforcer
+
+router = APIRouter()
+
+
+@router.post("/add_role_domain")
+async def add_role_domain(
+        request: Request,
+        data_in: schemas.AddRoleForUsersInDomain,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)):
+    """
+    在当前项目为角色添加相应权限
+    """
+
+    # username role dom
+    # for item in data_in.data:
+    #     is_exists_role = await crud.role.check(db, _id=item.role_id, game=item.game)
+    #     if not is_exists_role:
+    #         continue
+    #     casbin_enforcer.add_role_for_user_in_domain(user=item.username,
+    #                                                 role=item.role_id,
+    #                                                 domain=item.game)
+    #
+    # return schemas.Msg(code=0, msg='添加成功', data=True)
+    res = await crud.url_list.get_all(db)
+    role_id = {}
+    for i in res:
+        role_id[i['auth_id']] = i['name']
+    for item in data_in.data:
+        now_quanxian = await crud.user_url.get_quanxian(db, schemas.Url_quanxian(user_id=item.role_id))
+        # 如果不存在该用户其他游戏的权限，则新增一个
+        if now_quanxian == {}:
+            await crud.user_url.insert_quanxian(db, schemas.Url_quanxian(game=[item.game], user=item.username,
+                                                                         user_id=item.role_id,
+                                                                         quanxian=[role_id[item.auth_id]],
+                                                                         quanxian_id=[item.auth_id]))
+        # 存在则在这个用户加上要添加的游戏项目权限
+        else:
+            game = now_quanxian['game']
+            game.append(item.game)
+            quanxian = now_quanxian['quanxian']
+            quanxian.append(role_id[item.auth_id])
+            quanxian_id = now_quanxian['quanxian_id']
+            quanxian_id.append('auth_id')
+            await crud.user_url.updata_quanxian(db, schemas.Url_quanxian(game=game, user=item.username,
+                                                                         user_id=item.role_id, quanxian=quanxian,
+                                                                         quanxian_id=quanxian_id))
+    return schemas.Msg(code=0, msg='添加成功', data=True)
+
+
+@router.post("/get_permissions_for_user_in_domain")
+async def get_permissions_for_user_in_domain(
+        request: Request,
+        data_in: schemas.GetPermissionsForUserInDomain,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)):
+    """
+    获取域内用户或角色的权限
+    """
+    #data为列表
+    data = casbin_enforcer.get_permissions_for_user_in_domain(data_in.role_id, data_in.game)
+    paths = {i[2] for i in data}
+    #列表形式的coll_name
+    all_api = await crud.api_list.all_api(db)
+    for item in all_api:
+        if item['path'] in paths:
+            item['is_authz'] = True
+        else:
+            item['is_authz'] = False
+
+    return schemas.Msg(code=0, msg='ok', data=all_api)
+
+
+@router.post("/del_role_user_domain")
+async def del_role_domain(
+        request: Request,
+        data_in: schemas.DeleteRolesForUserInDomain,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)):
+    """
+    删除用户在当前项目中的权限
+    """
+
+    # username role dom
+    # res = casbin_enforcer.delete_roles_for_user_in_domain(user=data_in.username,
+    #                                                       role=data_in.role_id,
+    #                                                       domain=data_in.game)
+    #
+    # #await crud.role.delete_id(db, data_in.role_id)
+    # return schemas.Msg(code=0, msg='ok', data=res)
+    res = await crud.user_url.get_all(db)
+    for i in res:
+        if i['user'] == data_in.username:
+            for nu in range(len(i['game'])):
+                if i['game'][nu] == data_in.game:
+                    i['game'].remove(data_in.game)
+                    i['quanxian_id'].remove(i['quanxian_id'][nu])
+                    i['quanxian'].remove(data_in.role_id)
+                    await crud.user_url.updata_quanxian(db, schemas.Url_quanxian(game=i['game'], user=data_in.username,
+                                                                                 user_id=i['user_id'],
+                                                                                 quanxian_id=i['quanxian_id'],
+                                                                                 quanxian=i['quanxian']))
+    return schemas.Msg(code=0, msg='删除成功', data='')
+
+
+@router.post("/del_role_user")
+async def del_role_domain(
+        request: Request,
+        data_in: schemas.DeleteRolesForUserInDomain,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)):
+    """
+    删除角色管理板块中的角色
+    """
+    await crud.url_list.delete_name(db, data_in)
+    return schemas.Msg(code=0, msg="ok", data='')
+
+
+@router.post("/add_policy")
+async def add_policy(
+        request: Request,
+        data_in: schemas.Datalist,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)):
+    """
+    向当前权限添加新路由
+    """
+    # res = 0
+    # for path in data_id.path_list:
+    #     res = casbin_enforcer.add_policy(data_id.role_id, data_id.game, path, data_id.act)
+    # return schemas.Msg(code=0, msg='ok', data=res)
+    res = await crud.url_list.find_one_url(db, data_in)
+    for i in range(len(res['api_list'])):
+        if res['api_list'][i] == data_in.path:
+            res['state'][i] = True
+            await crud.url_list.update_url_url(db, res)
+    return schemas.Msg(code=0, msg='修改成功', data='')
+
+@router.post("/del_policy")
+async def remove_policy(
+        request: Request,
+        data_in: schemas.Del_role,
+        current_user: schemas.UserDB = Depends(deps.get_current_user)):
+    """
+    修改角色api权限
+    """
+    # res = casbin_enforcer.remove_policy(data_id.role_id, data_id.game, data_id.path, data_id.act)
+    # return schemas.Msg(code=0, msg='ok', data=res)
+    res = await crud.url_list.find_one_url(db, data_in)
+    for i in range(len(res['api_list'])):
+        if res['api_list'][i] == data_in.path:
+            res['state'][i] = False
+            await crud.url_list.update_url_url(db, res)
+    return schemas.Msg(code=0, msg='修改成功', data='')
+
+
+@router.post("/del_api_module")
+async def add_policy(
+        request: Request,
+        data_in: schemas.Add_module,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)):
+    res = await crud.api_module.get_one_module(db, data_in)
+    for i in range(len(res['state'])):
+        if data_in.url == res['api_list'][i]:
+            res['state'][i] = False
+            await crud.api_module.update_one_module(db, res)
+    return schemas.Msg(code=0, msg='修改成功', data='')
+
+
+@router.post("/add_api_module")
+async def add_policy(
+        request: Request,
+        data_in: schemas.Add_module,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)):
+    res = await crud.api_module.get_one_module(db, data_in)
+    for i in range(len(res['state'])):
+        if data_in.url == res['api_list'][i]:
+            res['state'][i] = True
+            await crud.api_module.update_one_module(db, res)
+    return schemas.Msg(code=0, msg='修改成功', data='')
+
+
+@router.get("/api_list")
+async def api_list(
+        request: Request,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)):
+    """
+
+    GetPermissionsForUserInDomain
+    所有的api
+    """
+    # res = await crud.api_list.all_api(db)
+    # return schemas.Msg(code=0, msg='ok', data=res)
+    re = await crud.api_module.get_api_module(db)
+    res = []
+    for i in re:
+        if i['path_name'] != 'root':
+            i['_id'] = str(i['_id'])
+            res.append(i)
+    return schemas.Msg(code=0, msg='ok', data=res)
+
+@router.post("/add_api")
+async def add_api(
+        request: Request,
+        data_in: schemas.AddApi,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """
+    添加api
+    """
+    # try:
+    #     res = await crud.api_list.add_api(db, data_in)
+    # except Exception as e:
+    #     return schemas.Msg(code=-1, msg='已经存在')
+    # return schemas.Msg(code=0, msg='ok', data=res.matched_count)
+    res = await crud.api_module.get_api_module(db)
+    for i in res:
+        if data_in.path in i['api_list']:
+            return schemas.Msg(code=0, msg='该路由已存在', data='')
+    path_list = []
+    for i in res:
+        path_list.append(i['path_name'])
+    if data_in.name in path_list:
+        for i in res:
+            if data_in.name == i['path_name']:
+                i['api_list'].append(data_in.path)
+                i['api_name'].append(data_in.desc)
+                i['state'].append(True)
+                await crud.api_module.updata_quanxian_module(db, schemas.Url_module(auth_id=i['auth_id'],
+                                                                                    path_name=data_in.name,
+                                                                                    api_list=i['api_list'],
+                                                                                    api_name=i['api_name'],
+                                                                                    state=i['state']))
+                return schemas.Msg(code=0, msg='ok', data='路由添加成功！')
+    else:
+        auth_list = []
+        for i in res:
+            auth_list.append(i['auth_id'])
+        auth_id = max(auth_list)
+        # api_data={}
+        # api_data['auth_id']='abc'+str(int(auth_id.split('c')[-1])+1)
+        # api_data['path_name']=data_in.name
+        # api_data['api_list']=[data_in.path]
+        # api_data['api_name']=[data_in.desc]
+        # api_data['state']=[True]
+        auth_id = 'abc' + str(int(auth_id.split('c')[-1]) + 1)
+        await crud.api_module.insert_quanxian(db, schemas.Url_module(auth_id=auth_id, path_name=data_in.name,
+                                                                     api_list=[data_in.path],
+                                                                     api_name=[data_in.desc], state=[True]))
+        return schemas.Msg(code=0, msg='ok', data='路由添加成功！')
+
+
+@router.post("/del_api")
+async def del_api(
+        request: Request,
+        data_in: schemas.DelApi,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)) -> schemas.Msg:
+    """
+    删除api
+    """
+    # 删除规则
+    paths = await crud.api_list.find_ids(db, data_in.ids, {'path': 1})
+    for item in paths:
+        casbin_enforcer.remove_filtered_policy(2, item['path'])
+
+    # 删除保存的记录
+    res = await crud.api_list.del_api(db, data_in)
+
+    return schemas.Msg(code=0, msg='ok', data=res.deleted_count)
+
+
+@router.post("/edit_api")
+async def edit_api(
+        request: Request,
+        data_in: schemas.EditApi,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)) -> schemas.Msg:
+    """
+    编辑api
+    """
+    res = await crud.api_list.edit_api(db, data_in)
+    return schemas.Msg(code=0, msg='ok', data=res.matched_count)
+
+
+@router.get("/domain")
+async def domain_list(
+        request: Request,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """
+    获取所有项目
+    """
+    # roel  dom path *
+    res = await crud.project.all_game(db)
+    return schemas.Msg(code=0, msg='ok', data=res)
+@router.get("/api_module")
+async def domain_list(
+        request: Request,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """
+    角色管理创建角色时显示的各个模块
+    """
+    res = await crud.api_module.get_api_module(db)
+    api_module=[]
+    for i in res:
+        if i['path_name'] !='root':
+            data=[]
+            data.append(i['auth_id'])
+            data.append(i['path_name'])
+            api_module.append(data)
+    return schemas.Msg(code=0, msg='ok', data=api_module)
+
+@router.post("/add_roles")
+async def add_roles(
+        request: Request,
+        game:str,
+        data_in: schemas.Add_role,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """
+    创建角色
+    """
+    # try:
+    #     res = await crud.role.add_role(db, data_in)
+    #     return schemas.Msg(code=0, msg='ok', data=res.upserted_id)
+    # except Exception as e:
+    #     return schemas.Msg(code=-1, msg='添加失败', data=str(e))
+    res = await crud.url_list.get_all(db)
+    for i in res:
+        if data_in.system == 1:
+            if data_in.name == i['name']:
+                return schemas.Msg(code=0, msg='该角色已存在！')
+        else:
+            if data_in.name == i['name'] and i['game'] == game:
+                return schemas.Msg(code=0, msg='该角色已存在！')
+    auth = []
+    if data_in.system == 1:
+        for i in res:
+            auth.append(i['auth_id'])
+        max_auth = 'ab' + str(int(max(auth).split('b')[-1]) + 1)
+        api_module = await crud.api_module.get_api_module(db)
+        for i in api_module:
+            if i['auth_id'] in data_in.path_name:
+                await crud.url_list.insert_url(db, schemas.Url_list(name=data_in.name, auth_id=max_auth,
+                                                                    path_name=i['path_name'], api_list=i['api_list'],
+                                                                    api_name=i['api_name'], state=i['state'],
+                                                                    system=data_in.system))
+            else:
+                state = []
+                for nu in range(len(i['state'])):
+                    state.append(False)
+                if i['path_name'] != 'root':
+                    await crud.url_list.insert_url(db, schemas.Url_list(name=data_in.name, auth_id=max_auth,
+                                                                        path_name=i['path_name'],
+                                                                        api_list=i['api_list'], api_name=i['api_name'],
+                                                                        state=state, system=data_in.system))
+        return schemas.Msg(code=0, msg='添加角色成功', data='')
+    else:
+        for i in res:
+            auth.append(i['auth_id'])
+        max_auth = 'ab' + str(int(max(auth).split('b')[-1]) + 1)
+        api_module = await crud.api_module.get_api_module(db)
+        for i in api_module:
+            if i['auth_id'] in data_in.path_name:
+                await crud.url_list.insert_urls(db, schemas.Url_lists(name=data_in.name, auth_id=max_auth,
+                                                                      path_name=i['path_name'], api_list=i['api_list'],
+                                                                      api_name=i['api_name'], state=i['state'],
+                                                                      system=data_in.system, game=game))
+            else:
+                state = []
+                for nu in range(len(i['state'])):
+                    state.append(False)
+                if i['path_name'] != 'root':
+                    await crud.url_list.insert_urls(db, schemas.Url_lists(name=data_in.name, auth_id=max_auth,
+                                                                          path_name=i['path_name'], game=game,
+                                                                          api_list=i['api_list'],
+                                                                          api_name=i['api_name'], state=state,
+                                                                          system=data_in.system))
+        return schemas.Msg(code=0, msg='添加角色成功', data='')
+
+@router.get("/roles")
+async def roles(
+        request: Request,
+        game: str,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """
+    获取所有的管理员用户
+    """
+    # res = await crud.role.dom_roles(db, game)
+    # return schemas.Msg(code=0, msg='ok', data=res)
+    res = await crud.url_list.get_all(db)
+    role = []
+    data = []
+    # 区分不同项目下的权限用户
+    for i in res:
+        if i['system'] == 1 and i['name'] != 'root':
+            role.append(i['name'])
+        if 'game' in i.keys():
+            if game == i['game']:
+                role.append(i['name'])
+    # 得到不同权限用户
+    role = list(set(role))
+    for id in role:
+        data_dcit = {}
+        data_dcit['name'] = id
+        auth_id = []
+        system = []
+        data_list = []
+        for i in res:
+            if i['name'] == id:
+                data_one = {}
+                auth_id.append(i['auth_id'])
+                system.append(i['system'])
+                data_one['path_name'] = i['path_name']
+                data_one['api_name'] = i['api_name']
+                data_one['api_list'] = i['api_list']
+                data_one['state'] = i['state']
+                data_list.append(data_one)
+        data_dcit['datalist'] = data_list
+        data_dcit['auth_id'] = auth_id[0]
+        data_dcit['system'] = system[0]
+        data.append(data_dcit)
+    return schemas.Msg(code=0, msg='ok', data=data)
+
+@router.post("/edit_role")
+async def edit_role(
+        request: Request,
+        date_in: schemas.Editname,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """
+    修改角色名
+    """
+    # res = await crud.role.edit_role(db, date_in)
+    # return schemas.Msg(code=0, msg='ok', data=res.matched_count)
+    await crud.url_list.edit_name(db,date_in)
+    return schemas.Msg(code=0,msg="ok")
+
+@router.get("/update_api_list")
+async def update_api_list(
+        request: Request,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user),
+):
+    """更新 api 列表"""
+    app = request.app
+    data = {}
+    for r in app.routes:
+        title = r.tags[0] if hasattr(r, 'description') else None
+        if not title:
+            continue
+        data.setdefault(title, {'list': []})
+        path = r.path
+        name = r.description if hasattr(r, 'description') else r.name
+        data[title]['list'].append({'api': path, 'title': name})
+
+    data = [{'title': k, 'list': v['list']} for k, v in data.items()]
+    for item in data:
+        title = item['title']
+        for l in item['list']:
+            api = l['api']
+            name = l['title']
+            add_data = schemas.UpdateApi(path=api, name=name)
+            await crud.api_list.update_api(db, add_data)
+
+    return schemas.Msg(code=0, msg='ok', data=1)
+
+
+@router.get("/account_owner_list")
+async def account_owner_list(request: Request,
+                             game: str,
+                             db: AsyncIOMotorDatabase = Depends(get_database),
+                             current_user: schemas.UserDB = Depends(deps.get_current_user)) -> schemas.Msg:
+    """获取账号owner权限"""
+    account_infos = await crud.user.find_many(db, {},
+                                              {'_id': False, 'name': True, 'nickname': True,
+                                               f'data_where.{game}': True})
+    resp = []
+    for account_info in account_infos:
+        resp.append(
+            {
+                'name': account_info.get('name'),
+                'nickname': account_info.get('nickname'),
+                'owner_list': ''
+            }
+        )
+        for item in account_info.get('data_where', {}).get(game, []):
+            if item.get('columnName') == 'owner_name':
+                resp[-1]['owner_list'] = ','.join(item.get('ftv', []))
+                break
+    return schemas.Msg(code=0, msg='ok', data=resp)
+# @router.post("/git_owner")
+# async def git_owner(request: Request,
+#                              game: str,
+#                              db: AsyncIOMotorDatabase = Depends(get_database),
+#                              current_user: schemas.UserDB = Depends(deps.get_current_user)) -> schemas.Msg:
+#     user=await crud.user
+
+@router.post("/update_account_owner")
+async def account_owner_list(request: Request,
+                             game: str,
+                             data_in: schemas.OwnerList,
+                             db: AsyncIOMotorDatabase = Depends(get_database),
+                             current_user: schemas.UserDB = Depends(deps.get_current_user)) -> schemas.Msg:
+    """设置账号owner权限"""
+    set_data = {
+        "columnName": "owner_name",
+        "tableType": "event",
+        "comparator": "in",
+        "ftv": data_in.owners
+    }
+    if not data_in.owners[0]:
+        res = await crud.user.update_one(db, {'name': data_in.account_name,
+                                              f'data_where.{game}': {'$exists': True}
+                                              },
+                                         {'$pull': {f'data_where.{game}': {'columnName': 'owner_name'}}}
+                                         )
+        return schemas.Msg(code=0, msg='ok', data=res.raw_result)
+
+    is_exists = await crud.user.find_one(db, {'name': data_in.account_name,
+                                              f'data_where.{game}': {'$exists': True},
+                                              })
+    if is_exists:
+        if await crud.user.find_one(db, {'name': data_in.account_name,
+                                         f'data_where.{game}': {'$exists': True},
+                                         f'data_where.{game}.columnName': 'owner_name'
+                                         }):
+            await crud.user.update_one(db, {'name': data_in.account_name,
+                                            f'data_where.{game}': {'$exists': True},
+                                            f'data_where.{game}.columnName': 'owner_name'
+                                            }, {'$set': {f'data_where.{game}.$': set_data}})
+        else:
+            await crud.user.update_one(db, {'name': data_in.account_name,
+                                            f'data_where.{game}': {'$exists': True},
+                                            }, {'$push': {f'data_where.{game}': set_data}})
+    else:
+        await crud.user.update_one(db, {'name': data_in.account_name,
+                                        }, {'$set': {f'data_where.{game}': [set_data]}})
+
+    return schemas.Msg(code=0, msg='ok')
+
+@router.get("/all_api_board")
+async def all_api_board(request: Request,
+                        db: AsyncIOMotorDatabase = Depends(get_database),
+                        current_user: schemas.UserDB = Depends(deps.get_current_user)) -> schemas.Msg:
+    """显示创建项目时生成的所有api权限模板"""
+    res = await crud.api_board.all_api(db)
+    for i in res:
+        i['_id'] = str(i['_id'])
+    return schemas.Msg(code=0, msg='ok', data=res)
+
+@router.post("/updata_api_board")
+async def updata_api_board(
+        request: Request,
+        opinion: bool,
+        data_in: schemas.Api_board,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)) -> schemas.Msg:
+    """
+    修改api权限模板
+    """
+    await crud.api_board.update(db, data_in,opinion)
+    return schemas.Msg(code=0, msg='ok')
+
+@router.post("/add_api_board")
+async def add_api_board(
+        request: Request,
+        data_in: schemas.Api_board,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)) -> schemas.Msg:
+    """
+    添加api权限模板
+    """
+    res = await crud.api_board.all_api(db)
+    for i in res:
+        if data_in.name ==i['name'] and data_in.api_name == i['api_name'] and data_in.api_path == i['api_path']:
+            return schemas.Msg(code=-1, msg='该路径已存在')
+    await crud.api_board.insert(db, data_in)
+    return schemas.Msg(code=0, msg='ok')
+
+@router.post("/del_api_board")
+async def del_api_board(
+        request: Request,
+        data_in: schemas.Api_board,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)) -> schemas.Msg:
+    """
+    删除api权限模板
+    """
+    await crud.api_board.del_api(db, data_in)
+    return schemas.Msg(code=0, msg='ok')

api/api_v1/check_data/__init__.py

@@ -0,0 +1 @@
+from .controller import router

api/api_v1/check_data/controller.py

@@ -0,0 +1,51 @@
+from fastapi import APIRouter, Request, Depends
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import schemas
+from api.api_v1.check_data import service
+from db import get_database
+
+router = APIRouter()
+
+
+@router.post("/check")
+async def check(request: Request,
+                data_in: schemas.CheckData,
+                game: str,
+                ) -> schemas.Msg:
+    res = await service.check_data(game, data_in)
+    return schemas.Msg(code=0, msg='ok', data=res)
+
+
+@router.post("/save")
+async def save(request: Request,
+               data_in: schemas.AddTemplate,
+               game: str,
+               db: AsyncIOMotorDatabase = Depends(get_database),
+               ) -> schemas.Msg:
+    res = await service.save_template(db, data_in, game)
+    return schemas.Msg(code=0, msg='ok', data=res)
+
+
+@router.get('/template')
+async def template(request: Request, game: str,
+                   db: AsyncIOMotorDatabase = Depends(get_database),
+
+                   ) -> schemas.Msg:
+    data = await service.get_template(db, game)
+    return schemas.Msg(code=0, msg='ok', data=data)
+
+
+@router.post('/del_template')
+async def del_template(request: Request, game: str, data_in: schemas.DelTemplate,
+                       db: AsyncIOMotorDatabase = Depends(get_database),
+
+                       ) -> schemas.Msg:
+    data = await service.del_template(db, data_in)
+    return schemas.Msg(code=0, msg='ok', data=data)
+
+
+@router.get('/default_field')
+async def template(request: Request, game: str) -> schemas.Msg:
+    data = service.get_default_field()
+    return schemas.Msg(code=0, msg='ok', data=data)

api/api_v1/check_data/service.py

@@ -0,0 +1,127 @@
+# coding:utf-8
+
+import copy
+import json
+import re
+from collections import namedtuple
+from ipaddress import IPv4Address
+
+import numpy as np
+
+import clickhouse_driver
+
+import schemas
+from core.config import settings
+from db import get_database
+from db.ckdb import ckdb as ck_client
+import crud
+
+Type = namedtuple('Type', ['string', 'integer', 'array', 'ipv4'])
+type_map = Type(string=str, integer=np.number, array=list, ipv4=IPv4Address)
+
+
+async def check_data(game, data_in: schemas.CheckData):
+    db = data_in.db_name
+    saixuan=data_in.game
+    event_name = data_in.event_name
+    is_unique = data_in.is_unique
+    props = data_in.props
+    where = data_in.where
+    limit = 5
+    check_type = copy.deepcopy(props)
+    check_type.update(data_in.default_field)
+
+    select = ','.join([f'`{field}`' for field in check_type.keys()])
+    if game == 'debug':
+        sql = f"""select {select} from {db}.event where game='{saixuan}' and `#event_name`='{event_name}'"""
+    else:
+        sql = f"""select {select} from {db}.event where game='{game}' and `#event_name`='{event_name}'"""
+    for k, v in where.items():
+        sql += f""" and `{k}`='{v}'"""
+
+    sql += f""" order by `#event_time` desc"""
+    sql += f""" limit {limit}"""
+
+    print(sql)
+    # pass_list: [], fail_list: []
+    # sql = 'show databases'
+    report = {'fail_list': [],
+              'pass_list': []}
+    fail_list = report['fail_list']
+    pass_list = report['pass_list']
+    try:
+        df = await ck_client.query_dataframe(sql)
+        report['title'] = df.columns.tolist()
+        report['data'] = []
+        for item in df.values:
+            report['data'].append([])
+            report['data'][-1] = [str(i) for i in item]
+
+    except clickhouse_driver.errors.ServerException as e:
+        if e.code == 47:
+            msg = re.match(r"""DB::Exception: Missing columns: '(.*)' while processing query""", e.message)
+            filed = '未知'
+            if msg:
+                filed = msg.group(1)
+            fail_list.append(f'<p style="color:red;font-size:17px;">数据库不存在字段-> {filed}</p>')
+        else:
+            fail_list.append('<p style="color:red;font-size:17px;">数据库查询未知错误</p>')
+
+        return report
+
+    if df.empty:
+        fail_list.append('<p style="color:blue;font-size:17px;">根据过滤条件未查到任何数据，也有可能是数据未及时入库。（3分钟后还没查到说明存在问题）</p>')
+        return report
+    if is_unique and len(df) > 1:
+        fail_list.append('<p style="color:yellow;font-size:17px;">警告：记录数大于一条</p>')
+
+    for k, t in check_type.items():
+        if t == 'json':
+
+            if isinstance(df[k][0], str):
+                try:
+                    json.loads(df[k][0])
+                    pass_list.append(f'<p style="color:green;font-size:17px;">通过：字段{k} 是期望的类型</p>')
+                    continue
+                except:
+                    fail_list.append(
+                        f"""<p style="color:red;font-size:17px;">错误：字段{k} 期望{t}类型，不是json格式</p>""")
+                    continue
+            else:
+                fail_list.append(
+                    f"""<p style="color:red;font-size:17px;">错误：字段{k} 期望{t}类型，得到{re.findall("'(.*)'>", str(type(df[k][0])))[0]}</p>""")
+                continue
+
+        if not isinstance(df[k][0], getattr(type_map, t)):
+            fail_list.append(
+                f"""<p style="color:red;font-size:17px;">错误：字段{k} 期望{t}类型，得到->{re.findall("'(.*)'>", str(type(df[k][0])))[0]}</p>""")
+        else:
+            pass_list.append(f'<p style="color:green;font-size:17px;">通过：字段{k} 是期望的类型</p>')
+
+    return report
+
+
+async def save_template(db, data_in: schemas.AddTemplate,
+                        game: str,
+                        ):
+    res = await crud.check_data.update_one(db, {'title': data_in.title, 'game': game},
+                                           {'$set': {'game': game,
+                                                     'check': data_in.check.dict()}},
+                                           upsert=True)
+    return True
+
+
+async def get_template(db, game, **kwargs):
+    res = []
+    async for doc in crud.check_data.find(db, {'game': game}, {'_id': False}, **kwargs):
+        res.append(doc)
+    return res
+
+
+def get_default_field():
+    return settings.DEFAULT_FIELD
+
+
+async def del_template(db, data_id: schemas.DelTemplate):
+    await crud.check_data.delete(db, data_id.dict())
+    return True

api/api_v1/ck_mana/event.py

@@ -0,0 +1,31 @@
+from collections import defaultdict
+
+import pandas as pd
+import numpy as np
+from fastapi import APIRouter, Depends, Request
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import crud, schemas
+from common import *
+
+from api import deps
+from db import get_database
+from db.ckdb import get_ck_db, CKDrive
+from db.redisdb import get_redis_pool, RedisDrive
+
+from models.behavior_analysis import BehaviorAnalysis
+from models.user_analysis import UserAnalysis
+from models.x_analysis import XAnalysis
+
+router = APIRouter()
+
+
+@router.post("/update_event_view")
+async def update_event_view(
+        request: Request,
+        game: str,
+        ckdb: CKDrive = Depends(get_ck_db),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """ 更新事件视图 """
+    pass

api/api_v1/endpoints/authority.py

@@ -0,0 +1,264 @@
+# import pymongo
+# from fastapi import APIRouter, Depends, Request
+# from motor.motor_asyncio import AsyncIOMotorDatabase
+# import crud, schemas
+# from core.config import settings
+# from core.security import get_password_hash
+#
+# from db import get_database
+# from api import deps
+# from db.ckdb import CKDrive, get_ck_db
+# from utils import casbin_enforcer
+#
+# router = APIRouter()
+#
+#
+# @router.get("/api_list")
+# async def api_list(request: Request,
+#                    current_user: schemas.UserDB = Depends(deps.get_current_user)) -> schemas.Msg:
+#     """api 列表"""
+#     app = request.app
+#     data = {}
+#     for r in app.routes:
+#         title = r.tags[0] if hasattr(r, 'description') else None
+#         if not title:
+#             continue
+#         data.setdefault(title, {'list': []})
+#         path = r.path
+#         name = r.description if hasattr(r, 'description') else r.name
+#         data[title]['list'].append({'api': path, 'title': name})
+#
+#     res = [{'title': k, 'list': v['list']} for k, v in data.items()]
+#
+#     return schemas.Msg(code=0, msg='ok', data=res)
+#
+#
+# @router.post('/set_data_auth')
+# async def set_data_auth(request: Request,
+#                         data_id: schemas.DataAuthSet,
+#                         game: str = Depends(deps.get_game_project),
+#                         db: AsyncIOMotorDatabase = Depends(get_database),
+#                         current_user: schemas.UserDB = Depends(deps.get_current_user)
+#                         ) -> schemas.Msg:
+#     """设置用户数据权限"""
+#     await crud.authority.set_data_auth(db, data_id, game=game)
+#     return schemas.Msg(code=0, msg='ok', data=data_id)
+#
+#
+# @router.get('/get_user_data_auth')
+# async def get_user_data_auth(request: Request,
+#                              game: str = Depends(deps.get_game_project),
+#                              db: AsyncIOMotorDatabase = Depends(get_database),
+#                              ck: CKDrive = Depends(get_ck_db),
+#                              current_user: schemas.UserDB = Depends(deps.get_current_user)
+#                              ) -> schemas.Msg:
+#     """获取当前用户数据权限"""
+#
+#     data_auth = await crud.authority.get_data_auth(db, username=request.user.name, game=game)
+#     if not data_auth:
+#         values = await ck.distinct(game, 'event', '#event_name')
+#         return schemas.Msg(code=0, msg='ok', data={
+#             'data': values,
+#             'game': game,
+#             'name': '全部事件'
+#         })
+#     data_auth_id = data_auth['data_auth_id']
+#     data = await crud.data_auth.get(data_auth_id)
+#     return schemas.Msg(code=0, msg='ok', data=data)
+#
+#
+# # @router.get('/get_users_data_auth')
+# # async def get_users_data_auth(request: Request,
+# #                               game: str = Depends(deps.get_game_project),
+# #                               db: AsyncIOMotorDatabase = Depends(get_database),
+# #                               ck: CKDrive = Depends(get_ck_db),
+# #                               current_user: schemas.UserDB = Depends(deps.get_current_user)
+# #                               ) -> schemas.Msg:
+# #     """获取当前项目所有用户数据权限"""
+# #
+# #     roles = await crud.authority.find_many(db, ptype='g', v2=game)
+# #     for item in roles:
+# #         user = item['v0']
+# #         data_auth = await crud.authority.get_data_auth(db, username=request.user.name, game=game)
+# #         if not data_auth:
+# #             values = await ck.distinct(game, 'event', '#event_name')
+# #             return schemas.Msg(code=0, msg='ok', data={
+# #                 'data': values,
+# #                 'game': game,
+# #                 'name': '全部事件'
+# #             })
+# #         data_auth_id = data_auth['data_auth_id']
+# #         data = await crud.data_auth.get(data_auth_id)
+# #         return schemas.Msg(code=0, msg='ok', data=data)
+# #
+# #     # data_auth = await crud.authority.get_data_auth(db, username=request.user.name, game=game)
+# #     # if not data_auth:
+# #     #     values = await ck.distinct(game, 'event', '#event_name')
+# #     #     return schemas.Msg(code=0, msg='ok', data={
+# #     #         'data': values,
+# #     #         'game': game,
+# #     #         'name': '全部事件'
+# #     #     })
+# #     # data_auth_id = data_auth['data_auth_id']
+# #     # data = await crud.data_auth.get(data_auth_id)
+# #     return schemas.Msg(code=0, msg='ok')
+#
+#
+# @router.post("/add_role")
+# async def add_role(request: Request,
+#                    data_in: schemas.CasbinRoleCreate,
+#                    game: str = Depends(deps.get_game_project),
+#                    db: AsyncIOMotorDatabase = Depends(get_database),
+#                    current_user: schemas.UserDB = Depends(deps.get_current_user)
+#                    ) -> schemas.Msg:
+#     """创建角色"""
+#
+#     # 不允许角色名和用户名一样
+#     if await crud.user.get_by_user(db, name=data_in.role_name):
+#         return schemas.Msg(code=-1, msg='请改个名字')
+#     role_dom = game
+#     api_dict = dict()
+#     for r in request.app.routes:
+#         api_dict[r.path] = r.description if hasattr(r, 'description') else r.name
+#     # 角色有的接口权限
+#     for obj in data_in.role_api:
+#         casbin_enforcer.add_policy(data_in.role_name, role_dom, obj, '*')
+#         await crud.authority.update_one(db, {'ptype': 'p', 'v0': data_in.role_name, 'v1': role_dom, 'v2': obj},
+#                                         {'$set': {'api_name': api_dict.get(obj)}})
+#
+#     # 管理员默认拥有该角色 方便从db中读出
+#     await crud.authority.create(db, 'g', settings.SUPERUSER_NAME, data_in.role_name, role_dom, '*',
+#                                 role_name=data_in.role_name,
+#                                 game=role_dom)
+#
+#     return schemas.Msg(code=0, msg='ok')
+#
+#
+# @router.post("/add_sys_role")
+# async def add_sys_role(request: Request,
+#                        data_in: schemas.CasbinRoleCreate,
+#                        game: str = Depends(deps.get_game_project),
+#                        db: AsyncIOMotorDatabase = Depends(get_database),
+#                        current_user: schemas.UserDB = Depends(deps.get_current_user)
+#                        ) -> schemas.Msg:
+#     """创建系统角色"""
+#     api_dict = dict()
+#
+#     # 不允许角色名和用户名一样
+#     if await crud.user.get_by_user(db, name=data_in.role_name):
+#         return schemas.Msg(code=-1, msg='请改个名字')
+#
+#     for r in request.app.routes:
+#         api_dict[r.path] = r.description if hasattr(r, 'description') else r.name
+#     # 角色有的接口权限
+#     for obj in data_in.role_api:
+#         casbin_enforcer.add_policy(data_in.role_name, '*', obj, '*')
+#         await crud.authority.create(db, 'p', data_in.role_name, '*', obj, '*', api_name=api_dict.get(obj))
+#
+#     # 管理员默认拥有该角色 方便从db中读出
+#     await crud.authority.create(db, 'g', settings.SUPERUSER_NAME, data_in.role_name,
+#                                 role_name=data_in.role_name,
+#                                 game='*')
+#
+#     return schemas.Msg(code=0, msg='ok')
+#
+#
+# @router.post("/add_account")
+# async def add_account(request: Request,
+#
+#                       data_in: schemas.AccountsCreate,
+#                       game: str = Depends(deps.get_game_project),
+#                       db: AsyncIOMotorDatabase = Depends(get_database),
+#                       current_user: schemas.UserDB = Depends(deps.get_current_user)
+#                       ) -> schemas.Msg:
+#     """添加账号"""
+#
+#     # 用户名不能与角色名重复
+#     roles = casbin_enforcer.get_all_roles()
+#     accounts = {item.username for item in data_in.accounts}
+#     # 用户名不能与已存在的重复
+#     exists_user = await crud.user.get_all_user(db)
+#     if accounts & set(roles) or accounts & set(exists_user):
+#         return schemas.Msg(code=-1, msg='已存在', data=list(set(accounts) & set(roles) | accounts & set(exists_user)))
+#
+#     """创建账号 并设置角色"""
+#     for item in data_in.accounts:
+#         account = schemas.UserCreate(name=item.username, password=settings.DEFAULT_PASSWORD)
+#         try:
+#             await crud.user.create(db, account)
+#         except pymongo.errors.DuplicateKeyError:
+#             return schemas.Msg(code=-1, msg='用户名已存在')
+#
+#         casbin_enforcer.add_grouping_policy(item.username, item.role_name, game)
+#         # 设置数据权限
+#         await crud.authority.set_data_auth(db,
+#                                            schemas.DataAuthSet(username=item.username, data_auth_id=item.data_auth_id),
+#                                            game)
+#
+#         # 添加到项目成员
+#     await crud.project.add_members(db, schemas.ProjectMember(project_id=data_in.project_id, members=list(accounts)))
+#
+#     return schemas.Msg(code=0, msg='ok')
+#
+#
+# @router.get("/all_role")
+# async def all_role(request: Request,
+#                    db: AsyncIOMotorDatabase = Depends(get_database),
+#                    game: str = Depends(deps.get_game_project),
+#                    current_user: schemas.UserDB = Depends(deps.get_current_user)
+#                    ) -> schemas.Msg:
+#     """获取所有角色"""
+#
+#     app = request.app
+#     api_data = {}
+#     for r in app.routes:
+#         title = r.tags[0] if hasattr(r, 'description') else None
+#         if not title:
+#             continue
+#         api_data[r.path] = {
+#             'api': r.path,
+#             'title': title,
+#             'name': r.description if hasattr(r, 'description') else r.name
+#         }
+#
+#     """获取域内所有角色"""
+#     roles = await crud.authority.find_many(db, {'role_name': {'$exists': 1}, 'game': game})
+#     dom_data = [{'role': item['v1'], 'title': item['role_name'], 'id': str(item['_id'])} for item in roles]
+#     for item in dom_data:
+#         q = await crud.authority.get_role_dom_authority(db, item['role'], game, api_data)
+#         item['authority'] = [{'title': k, 'child': v} for k, v in q.items()]
+#
+#     # 获取系统角色
+#     roles = await crud.authority.find_many(db, {'role_name':{'$exists': 1}, 'game':'*'})
+#     sys_data = [{'role': item['v1'], 'title': item['role_name'], 'id': str(item['_id'])} for item in roles]
+#     for item in sys_data:
+#         q = await crud.authority.get_role_dom_authority(db, item['role'], dom=game, api_data=api_data)
+#         item['authority'] = [{'title': k, 'child': v} for k, v in q.items()]
+#
+#     data = {
+#         'dom_role': dom_data,
+#         'sys_role': sys_data
+#     }
+#     return schemas.Msg(code=0, msg='ok', data=data)
+#
+# # @router.post("/set_role")
+# # async def set_role(request: Request,
+# #                    data_id: schemas.AccountSetRole,
+# #                    db: AsyncIOMotorDatabase = Depends(get_database),
+# #                    current_user: schemas.UserDB = Depends(deps.get_current_user)
+# #                    ) -> schemas.Msg:
+# #     """设置账号角色"""
+# #     casbin_enforcer.delete_user(data_id.name)
+# #     casbin_enforcer.add_role_for_user(data_id.name, data_id.role_name)
+# #     await crud.authority.update_one(db, {'ptype': 'g', 'v0': data_id.name}, dict(v1=data_id.role_name))
+# #
+# #     return schemas.Msg(code=0, msg='ok')
+#
+# # @router.get("/delete_user")
+# # async def delete_user(request: Request,
+# #                       data_id: schemas.AccountDeleteUser,
+# #                       db: AsyncIOMotorDatabase = Depends(get_database),
+# #                       current_user: schemas.UserDB = Depends(deps.get_current_user)
+# #                       ) -> schemas.Msg:
+# #     pass
+# #     return schemas.Msg(code=0, msg='暂时没有')

api/api_v1/endpoints/dashboard.py

@@ -0,0 +1,260 @@
+import pymongo
+from fastapi import APIRouter, Depends, Request
+from motor.motor_asyncio import AsyncIOMotorDatabase
+import crud, schemas
+
+from db import get_database
+from api import deps
+from utils.func import get_uid
+
+router = APIRouter()
+
+
+@router.post("/create")
+async def create(
+        data_in: schemas.DashboardCreate,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """创建看板"""
+    try:
+        await crud.dashboard.create(db, data_in, user_id=current_user.id)
+    except pymongo.errors.DuplicateKeyError:
+        return schemas.Msg(code=-1, msg='看板已存在', data='看板已存在')
+
+    return schemas.Msg(code=0, msg='ok', data='创建成功')
+
+
+@router.post('/edit_show_report')
+async def edit_show_report(
+        request: Request,
+        data_in: schemas.EditShowReport,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    report_id = data_in.config.report_id
+    res = await crud.dashboard.update_one(db, {'_id': data_in.dashboard_id, 'reports.report_id': report_id},
+                                          {'$set': {f'reports.$.{k}': v for k, v in
+                                                    data_in.config.dict(skip_defaults=True).items()}})
+    if res.modified_count == 1:
+        return schemas.Msg(code=0, msg='ok', data=data_in.config)
+    elif res.modified_count == 0:
+        return schemas.Msg(code=-1, msg='没有修改', data=dict())
+
+
+@router.post("/delete")
+async def delete(
+        request: Request,
+        data_in: schemas.DashboardDelete,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """删除看板"""
+    del_dashboard = await crud.dashboard.delete(db, {'_id': {'$in': data_in.ids}})
+
+    if del_dashboard.deleted_count == 0:
+        return schemas.Msg(code=-1, msg='error', data='删除失败')
+    return schemas.Msg(code=0, msg='ok', data='删除成功')
+
+
+@router.post("/move")
+async def move(
+        data_in: schemas.DashboardMove,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """
+    移动看板
+    """
+    for source_id in data_in.source_ids:
+        res = await crud.dashboard.update_one(db, {'_id': source_id},
+                                              {'$set': dict(cat=data_in.cat, pid=data_in.dest_pid)})
+    return schemas.Msg(code=0, msg='ok', data='移动成功')
+
+
+@router.post("/sort")
+async def sort(
+        game: str,
+        data_in: schemas.DashboardSort,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """
+    看板排序
+    """
+    for item in data_in.sort:
+        await crud.dashboard.set_sort(db, index=item.dashboard_id, sort=item.sort)
+    return schemas.Msg(code=0, msg='ok', data=1)
+
+
+@router.post("/copy")
+async def copy(
+        request: Request,
+        data_in: schemas.DashboardCopy,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """
+    复制到其他项目
+    """
+    # 检查是否存在默认空间 不存在就创建
+    dest_project_id = data_in.dest_project_id
+    dest_default_space = await crud.space.find_one(db, {'project_id': dest_project_id, 'name': '默认空间'},
+                                                   {'_id': True})
+    dest_space_id = dest_default_space.get('_id')
+    user_id = request.user.id
+    # 创建默认空间
+    if not dest_space_id:
+        default_space = await crud.space.create(db,
+                                                schemas.SpaceCreate(name='默认空间', project_id=dest_project_id),
+                                                user=current_user)
+        dest_space_id = default_space.inserted_id
+
+    dashboards = await crud.dashboard.find_many(db, {'_id': {'$in': data_in.source_ids}}, {'_id': False})
+    for item in dashboards:
+        item['project_id'] = dest_project_id
+        item['pid'] = dest_space_id
+        item['cat'] = 'space'
+        item['user_id'] = user_id
+        item['_id'] = get_uid()
+        for report in item['reports']:
+            report_id = report['report_id']
+            new_report = await crud.report.get(db, report_id)
+            new_report_id = get_uid()
+            report['report_id'] = new_report_id
+            new_report['user_id'] = user_id
+            new_report['_id'] = new_report_id
+            new_report['project_id'] = dest_project_id
+            try:
+                await crud.report.insert_one(db, new_report)
+            except:
+                exists_report = await crud.report.find_one(db, {'project_id': item['project_id'],
+                                                                'user_id': item['user_id'], 'name': report['name']})
+                report['report_id'] = exists_report['_id']
+        try:
+            await crud.dashboard.update_one(db,
+                                            {'project_id': item['project_id'], 'name': item['name'],
+                                             'user_id': item['user_id']}, {'$set': item},
+                                            upsert=True)
+        except:
+            pass
+
+    return schemas.Msg(code=0, msg='ok', data='复制成功')
+
+
+@router.post("/copy_to_my_space")
+async def copy(
+        request: Request,
+        data_in: schemas.DashboardCopyToSpace,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """
+    复制到自己空间
+    """
+    # 检查是否存在默认空间 不存在就创建
+    dest_project_id = data_in.project_id
+    dest_space_id = data_in.dest_space_id
+    user_id = request.user.id
+
+    dashboards = await crud.dashboard.find_many(db, {'_id': {'$in': data_in.source_ids}}, {'_id': False})
+    for item in dashboards:
+        item['project_id'] = dest_project_id
+        item['pid'] = dest_space_id
+        item['user_id'] = user_id
+        item['cat'] = 'space'
+        item['_id'] = get_uid()
+        for report in item['reports']:
+            report_id = report['report_id']
+            new_report = await crud.report.get(db, report_id)
+            new_report_id = get_uid()
+            report['report_id'] = new_report_id
+            new_report['_id'] = new_report_id
+            new_report['user_id'] = user_id
+            new_report['project_id'] = dest_project_id
+            try:
+                await crud.report.insert_one(db, new_report)
+            except:
+                exists_report = await crud.report.find_one(db, {'project_id': item['project_id'],
+                                                                'user_id': item['user_id'], 'name': report['name']})
+                report['report_id'] = exists_report['_id']
+        try:
+            await crud.dashboard.update_one(db,
+                                            {'project_id': item['project_id'], 'name': item['name'],
+                                             'user_id': item['user_id']}, {'$set': item},
+                                            upsert=True)
+        except:
+            pass
+
+    return schemas.Msg(code=0, msg='ok', data='复制成功')
+
+
+@router.post("/add_report")
+async def add_report(data_in: schemas.AddReport,
+                     game: str,
+                     db: AsyncIOMotorDatabase = Depends(get_database),
+                     current_user: schemas.UserDB = Depends(deps.get_current_user)
+                     ):
+    """添加报表"""
+    reports = [item.dict() for item in data_in.report_ids]
+    # res = await crud.dashboard.update_one(db, {'_id': data_in.id},
+    #                                       {'$push': {'reports': {'$each': reports}}})
+    await crud.dashboard.update_one(db, {'_id': data_in.id},
+                                    {'$set': {'reports': reports}})
+    return schemas.Msg(code=0, msg='ok', data='ok')
+
+
+@router.post("/edit_report")
+async def edit_report(data_in: schemas.EditReport,
+                      game: str,
+                      db: AsyncIOMotorDatabase = Depends(get_database),
+                      current_user: schemas.UserDB = Depends(deps.get_current_user)
+                      ):
+    """看板样式设置"""
+
+    res = await crud.dashboard.update_one(db, {'_id': data_in.id, 'reports.report_id': data_in.report.report_id},
+                                          {'$set': {f'reports.$.{k}': v for k, v in
+                                                    data_in.report.dict(skip_defaults=True).items()}})
+
+    return schemas.Msg(code=0, msg='ok', data='ok')
+
+
+@router.post("/del_report")
+async def del_report(
+        game: str,
+        data_in: schemas.DelReport,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+):
+    """删除报表"""
+    del_item = {'report_id': data_in.report_id}
+    await crud.dashboard.update_one(db, {'_id': data_in.id}, {'$pull': {'reports': del_item}})
+    return schemas.Msg(code=0, msg='ok', data='ok')
+
+
+@router.post("/edit")
+async def edit(
+        game: str,
+        data_in: schemas.EditDashboard,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+):
+    """编辑看板名"""
+    await crud.dashboard.update_one(db, {'_id': data_in.dashboard_id}, {'$set': {'name': data_in.new_name}})
+    return schemas.Msg(code=0, msg='ok', data='ok')
+
+
+@router.post("/")
+async def dashboards(request: Request,
+                     game: str,
+                     data_in: schemas.ReadDashboard,
+                     db: AsyncIOMotorDatabase = Depends(get_database),
+                     current_user: schemas.UserDB = Depends(deps.get_current_user)
+                     ):
+    """获取一个看板"""
+    res = await crud.dashboard.get(db, id=data_in.id)
+    reports = {item['report_id']: item for item in res['reports']}
+    reports_detail = await crud.report.find_many(db, {'_id': {'$in': list(reports.keys())}}, {'query.cachedata': False})
+    for item in reports_detail:
+        reports[item['_id']].update(item)
+    return schemas.Msg(code=0, msg='ok', data=reports)

api/api_v1/endpoints/data_auth.py

@@ -0,0 +1,306 @@
+import json
+
+import pymongo
+from bson import ObjectId
+from fastapi import APIRouter, Depends, Request
+from fastapi.encoders import jsonable_encoder
+from motor.motor_asyncio import AsyncIOMotorDatabase
+from redis import Redis
+
+import crud, schemas
+from core.config import settings
+from core.security import get_password_hash
+
+from db import get_database
+from api import deps
+from db.ckdb import CKDrive, get_ck_db
+from db.redisdb import get_redis_pool, RedisDrive
+
+# from utils import casbin_enforcer
+
+router = APIRouter()
+
+
+@router.post('/add_data_auth')
+async def add_data_auth(request: Request,
+                        data_id: schemas.DataAuthCreate,
+                        game: str = Depends(deps.get_game_project),
+                        db: AsyncIOMotorDatabase = Depends(get_database),
+                        current_user: schemas.UserDB = Depends(deps.get_current_user)
+                        ) -> schemas.Msg:
+    """创建数据权限"""
+    await crud.data_auth.create(db, data_id, game)
+    return schemas.Msg(code=0, msg='ok', data=data_id)
+
+
+@router.post('/edit_data_auth')
+async def edit_data_auth(request: Request,
+                         data_id: schemas.DataAuthEdit,
+                         game: str,
+                         db: AsyncIOMotorDatabase = Depends(get_database),
+                         current_user: schemas.UserDB = Depends(deps.get_current_user)
+                         ) -> schemas.Msg:
+    """修改数据权限"""
+    await crud.data_auth.edit_data_auth(db, data_id)
+    return schemas.Msg(code=0, msg='ok', data=data_id)
+
+
+@router.get("/quotas_map")
+async def quotas_map(
+        request: Request,
+        game: str,
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    return schemas.Msg(code=0, msg='ok', data=settings.CK_OPERATOR)
+
+
+@router.get("/filter_map")
+async def filter_map(
+        request: Request,
+        game: str,
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    return schemas.Msg(code=0, msg='ok', data=settings.CK_FILTER)
+
+
+@router.get('/all_event')
+async def all_event(request: Request,
+                    game: str,
+                    ck: CKDrive = Depends(get_ck_db),
+                    db: AsyncIOMotorDatabase = Depends(get_database),
+                    current_user: schemas.UserDB = Depends(deps.get_current_user)
+                    ) -> schemas.Msg:
+    """获取所有事件"""
+    values = await ck.distinct(game, 'event', '#event_name')
+    values.sort()
+    return schemas.Msg(code=0, msg='ok', data=values)
+
+
+@router.get("/list")
+async def data_authority(request: Request,
+                         game: str,
+                         db: AsyncIOMotorDatabase = Depends(get_database),
+                         rdb: RedisDrive = Depends(get_redis_pool),
+                         ck: CKDrive = Depends(get_ck_db),
+                         current_user: schemas.UserDB = Depends(deps.get_current_user)
+                         ) -> schemas.Msg:
+    """获取前项目数据权限"""
+    total_event = await ck.distinct_count(game, 'event', '#event_name')
+    data = await crud.data_auth.get_game_data_auth(db, game)
+    for item in data:
+        item['id'] = str(item['_id'])
+        del item['_id']
+        item['data_range'] = f'{len(item["data"])}/{total_event}'
+
+    data = jsonable_encoder(data)
+    return schemas.Msg(code=0, msg='ok', data=data)
+
+
+@router.get("/my_event")
+async def my_event(request: Request,
+                   game: str,
+                   db: AsyncIOMotorDatabase = Depends(get_database),
+                   rdb: RedisDrive = Depends(get_redis_pool),
+                   ck: CKDrive = Depends(get_ck_db),
+                   current_user: schemas.UserDB = Depends(deps.get_current_user)
+                   ) -> schemas.Msg:
+    """获取自己的事件权限"""
+    event_list = []
+    # start_date = (datetime.datetime.now()-datetime.timedelta(days=30)).strftime('%Y-%m-%d %H:%M:%S')
+    # where = f"""`#event_time` > '{start_date}'"""
+    # my_data_auth = await ck.distinct(game, 'event', '#event_name',where)
+    my_data_auth = await rdb.smembers(f'{game}_event_set')
+
+    #
+    # else:
+    #     # 设置了数据权限
+    #     my_data_auth = await crud.data_auth.get(db, ObjectId(data_auth_id))
+    #     my_data_auth = my_data_auth['data']
+
+    event_show_name = await crud.event_mana.get_all_show_name(db, game)
+    event_list.append({'id': 'event', 'title': '全部事件', 'category': []})
+    for item in my_data_auth:
+        event_list[-1]['category'].append({
+            'event_name': item,
+            'event_desc': event_show_name.get(item, item)
+        })
+    event_list[-1]['category'].append({'event_name': '*', 'event_desc': '任意事件'})
+    event_list.sort()
+    return schemas.Msg(code=0, msg='ok', data=event_list)
+
+
+@router.get("/user_property")
+async def user_property(request: Request,
+                        game: str,
+                        db: AsyncIOMotorDatabase = Depends(get_database),
+                        rdb: RedisDrive = Depends(get_redis_pool),
+                        ck: CKDrive = Depends(get_ck_db),
+                        current_user: schemas.UserDB = Depends(deps.get_current_user)
+                        ) -> schemas.Msg:
+    """获取用户属性"""
+    data = await rdb.get(f'{game}_user')
+    data = json.loads(data)
+    propertys = []
+
+    data_attr = await crud.data_attr.find_many(db, {'game': game, 'cat': 'user'})
+    data_attr = {item['name']: item for item in data_attr}
+
+    for k, v in data.items():
+        data_type = settings.CK_TYPE_DICT.get(v)
+        propertys.append(
+            {'name': k,
+             'data_type': data_type,
+             'show_name': data_attr.get(k, {}).get('show_name', ''),
+             }
+        )
+    propertys = sorted(propertys, key=lambda x: x['show_name'])
+
+    return schemas.Msg(code=0, msg='ok', data=propertys)
+
+
+@router.post('/load_prop_quotas')
+async def load_prop_quotas(request: Request,
+                           game: str,
+                           data_in: schemas.LoadProQuotas,
+                           db: AsyncIOMotorDatabase = Depends(get_database),
+                           rdb: RedisDrive = Depends(get_redis_pool),
+                           ck: CKDrive = Depends(get_ck_db),
+                           current_user: schemas.UserDB = Depends(deps.get_current_user)
+                           ) -> schemas.Msg:
+    """事件属性 聚合条件"""
+
+    event_columns = await ck.get_columns(game, 'event')
+
+    data_attr = await crud.data_attr.find_many(db, {'game': game, 'cat': 'event'})
+    data_attr = {item['name']: item for item in data_attr}
+    event_props = []
+    for item in event_columns:
+        data_type = settings.CK_TYPE_DICT.get(item['type'])
+        title = data_attr.get(item['name'], {}).get('show_name') or item['name']
+        event_prop = {
+            'id': item['name'],
+            'data_type': data_type,
+            'title': title,
+            # 'category': settings.CK_OPERATOR.get(data_type) or []
+        }
+        event_props.append(event_prop)
+    if data_in.model == 'scatter':
+        staid_quots = [
+            {
+                "id": "*",
+                "data_type": None,
+                "analysis": "times",
+                "title": "次数",
+            },
+            {
+                "id": "*",
+                "data_type": None,
+                "analysis": "number_of_days",
+                "title": "天数",
+            },
+            {
+                "id": "*",
+                "data_type": None,
+                "analysis": "number_of_hours",
+                "title": "小时数",
+            },
+        ]
+    else:
+        staid_quots = [
+            {
+                "id": "*",
+                "data_type": None,
+                "analysis": "total_count",
+                "title": "总次数",
+            },
+            {
+                "id": "*",
+                "analysis": "touch_user_count",
+                "data_type": None,
+                "title": "触发用户数",
+            },
+            {
+                "id": "*",
+                "analysis": "touch_user_avg",
+                "data_type": None,
+                "title": "人均次数",
+            },
+        ]
+
+    res = {
+        'props': event_props,
+        'staid_quots': staid_quots
+
+    }
+
+    return schemas.Msg(code=0, msg='ok', data=res)
+
+
+@router.post('/load_filter_props')
+async def load_filter_props(request: Request,
+                            game: str,
+                            data_in: schemas.LoadProQuotas,
+                            db: AsyncIOMotorDatabase = Depends(get_database),
+                            rdb: RedisDrive = Depends(get_redis_pool),
+                            ck: CKDrive = Depends(get_ck_db),
+                            current_user: schemas.UserDB = Depends(deps.get_current_user)
+                            ) -> schemas.Msg:
+    """事件属性 过滤条件"""
+
+    event_columns = await ck.get_columns(game, 'event')
+    user_columns = await ck.get_columns(game, 'user')
+
+    data_attr = await crud.data_attr.find_many(db, {'game': game, 'cat': 'event'})
+    data_attr = {item['name']: item for item in data_attr}
+    event_props = []
+    for item in event_columns:
+        data_type = settings.CK_TYPE_DICT.get(item['type'])
+        title = data_attr.get(item['name'], {}).get('show_name') or item['name']
+        event_prop = {
+            'id': item['name'],
+            'data_type': data_type,
+            'title': title,
+        }
+        event_props.append(event_prop)
+
+    data_attr = await crud.data_attr.find_many(db, {'game': game, 'cat': 'user'})
+    data_attr = {item['name']: item for item in data_attr}
+    user_props = []
+    for item in user_columns:
+        data_type = settings.CK_TYPE_DICT.get(item['type'])
+        title = data_attr.get(item['name'], {}).get('show_name') or item['name']
+        user_prop = {
+            'id': item['name'],
+            'data_type': data_type,
+            'title': title,
+        }
+        user_props.append(user_prop)
+
+    user_label_props = []
+    user_label_docs = await crud.user_label.find_many(db, {'game': game}, {'qp': 0})
+    for item in user_label_docs:
+        tmp = {
+            'id': item['cluster_name'],
+            'data_type': 'user_label',
+            'title': item['display_name'],
+        }
+        user_label_props.append(tmp)
+    res = [
+        {
+            'title': '事件属性',
+            'id': 'event',
+            'category': event_props
+        },
+        {
+            'title': '用户属性',
+            'id': 'user',
+            'category': user_props
+        },
+        {
+            'title': '用户标签',
+            'id': 'user_label',
+            'category': user_label_props
+        }
+    ]
+
+    return schemas.Msg(code=0, msg='ok', data=res)

api/api_v1/endpoints/data_mana.py

@@ -0,0 +1,217 @@
+import json
+
+from aioredis import Redis
+from fastapi import APIRouter, Depends, Request, File
+from motor.motor_asyncio import AsyncIOMotorDatabase
+import pandas as pd
+
+import crud, schemas
+
+from api import deps
+from core.config import settings
+from db import get_database
+from db.ckdb import CKDrive, get_ck_db
+from db.redisdb import get_redis_pool
+from utils import estimate_data,dict_to_str
+
+router = APIRouter()
+
+__all__ = 'router',
+
+
+@router.get("/attr_list")
+async def read_data_attr(
+        request: Request,
+        game: str,
+        cat: str,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        rdb: Redis = Depends(get_redis_pool),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """事件属性列表或用户属性列表"""
+    data = await rdb.get(f'{game}_{cat}')
+    data = json.loads(data)
+    res = []
+
+    data_attr = await crud.data_attr.find_many(db, {'game': game, 'cat': cat})
+    data_attr = {item['name']: item for item in data_attr}
+
+    for k, v in data.items():
+        res.append(
+            {'name': k,
+             'data_type': settings.CK_TYPE_DICT.get(v),
+             'show_name': data_attr.get(k, {}).get('show_name', ''),
+             'is_show': data_attr.get(k, {}).get('is_show', True),
+             'attr_type': '预置属性' if k.startswith('#') else '自定义属性',
+             'unit': ''
+             }
+        )
+    return schemas.Msg(code=0, msg='ok', data=res)
+@router.post("/game_user_event_list")
+async def read_data_attr(
+        request: Request,
+        game: str,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        ck: CKDrive = Depends(get_ck_db),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """用户搜索时显示的用户属性"""
+    # data = await rdb.get(f'{game}_{data_in.cat}')
+    # data = json.loads(data)
+    # res = list(data.keys())
+    #event_columns = await ck.get_columns(game, 'event')
+    user_columns = await ck.get_columns(game, 'user')
+    data_attr = await crud.data_attr.find_many(db, {'game': game, 'cat': 'user'})
+    data_attr = {item['name']: item for item in data_attr}
+    user_props = []
+    for item in user_columns:
+        data_type = settings.CK_TYPE_DICT.get(item['type'])
+        title = data_attr.get(item['name'], {}).get('show_name') or item['name']
+        user_prop = {
+            'id': item['name'],
+            'data_type': data_type,
+            'title': title,
+        }
+        user_props.append(user_prop)
+
+    user_label_props = []
+    user_label_docs = await crud.user_label.find_many(db, {'game': game}, {'qp': 0})
+    for item in user_label_docs:
+        tmp = {
+            'id': item['cluster_name'],
+            'data_type': 'user_label',
+            'title': item['display_name'],
+        }
+        user_label_props.append(tmp)
+    res = [
+        {
+            'title': '用户属性',
+            'id': 'user',
+            'category': user_props
+        },
+        {
+            'title': '用户标签',
+            'id': 'user_label',
+            'category': user_label_props
+        }
+    ]
+    return schemas.Msg(code=0, msg='ok', data=res)
+
+@router.post("/attr_edit")
+async def edit_data_attr(
+        request: Request,
+        game: str,
+        data_in: schemas.DataAttrEdit,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        rdb: Redis = Depends(get_redis_pool),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """编辑事件属性"""
+    await crud.data_attr.edit_data_attr(db, game, data_in)
+    return schemas.Msg(code=0, msg='ok', data=data_in)
+
+
+
+# @router.post("/add_select_map")
+# async def add_map(
+#         request: Request,
+#         game: str,
+#         data_in: schemas.SelectMap,
+#         db: AsyncIOMotorDatabase = Depends(get_database),
+#         current_user: schemas.UserDB = Depends(deps.get_current_user)
+# ) -> schemas.Msg:
+#     """添加属性值选择映射"""
+#
+#     """
+#     {
+#     game:aaa,
+#     attr_name:bbb,
+#     map_:{
+#     '区服aa':'1',
+#     '区服vvv':'5',
+#     }
+#
+#     }
+#     """
+#     await crud.select_map.save(db, data_in)
+#     return schemas.Msg(code=0, msg='ok', data=data_in)
+
+#在gametoos同步区服了，所以不需要这段代码
+@router.post("/add_select_map")
+async def add_select_map(
+        request: Request,
+        game: str,
+        file: bytes = File(...),
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """添加游戏区服信息选择映射"""
+    dfs = pd.read_excel(file, engine='openpyxl', sheet_name=None)
+    for attr_name, df in dfs.items():
+        #将id这列转换成字符串类型
+        if len(df) >0:
+            df['id'] = df['id'].astype(str)
+            map_ = df.to_dict('records')
+            data_in = schemas.SelectMap(game=game, attr_name=attr_name, map_=map_)
+            await crud.select_map.save(db, data_in)
+    return schemas.Msg(code=0, msg='ok', data=1)
+
+
+@router.get("/select_list")
+async def select_list(
+        request: Request,
+        game: str,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """属性值选择映射列表"""
+    #当游戏为魔法门H5时，把game的值改为数据库中对应的值（mdb中的值和ck中的值是不一样的）
+    if game == 'mfmh5':
+        game='mzmfmh5'
+    resp = await crud.select_map.get_list(db, game)
+    return schemas.Msg(code=0, msg='ok', data=resp)
+
+
+@router.post("/select_attr")
+async def select_attr(
+        request: Request,
+        game: str,
+        data_in: schemas.SelectAttr,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """属性值选择映射"""
+    resp = await crud.select_map.get_select(db, data_in, game)
+    code = 0 if resp else -9
+    if resp:
+        if 'map_' in resp.keys():
+            return schemas.Msg(code=code, msg='ok', data=resp)
+        else:
+            resp['map_'] = resp.pop('owner_name')
+            return schemas.Msg(code=code, msg='ok', data=resp)
+    else:
+        return schemas.Msg(code=code, msg='ok', data=resp)
+
+@router.post("/add_attr")
+async def add_attr(
+        request: Request,
+        game: str,
+        data_in: schemas.Add_attr,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        rdb: Redis = Depends(get_redis_pool),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """添加事件属性或添加用户属性"""
+    data = await rdb.get(f'{game}_{data_in.cat}')
+    data = json.loads(data)
+    if data_in.state =='add':
+        #判断传入数据类型
+        new_data_type=estimate_data(data_in.data_type)
+        #添加数据
+        data[data_in.new_attribute]=new_data_type
+    else:
+        del data[data_in.new_attribute]
+    #将字典转为字符串
+    str_data=dict_to_str(data)
+    await rdb.set(f'{game}_{data_in.cat}',str_data)
+    return schemas.Msg(code=0, msg='ok')

api/api_v1/endpoints/event_mana.py

@@ -0,0 +1,64 @@
+import json
+
+from aioredis import Redis
+from fastapi import APIRouter, Depends, Request
+from motor.motor_asyncio import AsyncIOMotorDatabase
+import pandas as pd
+
+import crud, schemas
+
+from api import deps
+from core.config import settings
+from db import get_database
+from db.ckdb import CKDrive, get_ck_db
+from db.redisdb import get_redis_pool
+
+router = APIRouter()
+
+__all__ = 'router',
+
+
+@router.get("/event_list")
+async def event_list(
+        request: Request,
+        game: str,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        ckdb: CKDrive = Depends(get_ck_db),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """事件列表"""
+    #获取事件名
+    try:
+        event_list = await ckdb.distinct(game, 'event', '#event_name')
+        # 获取事件量
+        event_count = await ckdb.yesterday_event_count(game)
+        event_meta = await crud.event_mana.find_many(db, {'game': game}) or {}
+    except Exception as e:
+        return schemas.Msg(code=-9, msg='查无数据', data='')
+    if event_meta:
+        event_meta = pd.DataFrame(event_meta).set_index('event_name').fillna('').T.to_dict()
+
+    res = []
+    for name in event_list:
+        res.append({
+            'name': name,
+            'show_name': event_meta.get(name, {}).get('show_name', ''),
+            'is_show': event_meta.get(name, {}).get('is_show', True),
+            'desc': event_meta.get(name, {}).get('desc', ''),
+            'event_count': event_count.get(name, {}).get('v')
+        }
+        )
+    return schemas.Msg(code=0, msg='ok', data=res)
+
+
+@router.post("/event_edit")
+async def event_edit(
+        request: Request,
+        game: str,
+        data_in: schemas.EventMateEdit,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """编辑事件"""
+    await crud.event_mana.edit_event_mate(db, game, data_in)
+    return schemas.Msg(code=0, msg='ok', data=data_in)

api/api_v1/endpoints/folder.py

@@ -0,0 +1,40 @@
+import pymongo
+from fastapi import APIRouter, Depends
+from motor.motor_asyncio import AsyncIOMotorDatabase
+import crud, schemas
+
+from db import get_database
+from api import deps
+
+router = APIRouter()
+
+
+@router.post("/create")
+async def create(
+        data_in: schemas.FolderCreate,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """创建文件夹"""
+    try:
+        await crud.folder.create(db, data_in, user_id=current_user.id)
+    except pymongo.errors.DuplicateKeyError:
+        return schemas.Msg(code=-1, msg='文件夹已存在', data='文件夹已存在')
+
+    return schemas.Msg(code=0, msg='ok', data='创建成功')
+
+
+@router.post("/delete")
+async def delete(
+        data_in: schemas.FolderDelete,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """删除文件夹"""
+    # 删除文件夹 自己创建的
+    del_folder = await crud.folder.delete(db, _id=data_in.id, user_id=current_user.id)
+    # 删除文件夹下的 dashboard
+    del_dashboard = await crud.dashboard.delete(db, pid=data_in.id)
+    if del_folder.deleted_count == 0:
+        return schemas.Msg(code=-1, msg='error', data='删除失败')
+    return schemas.Msg(code=0, msg='ok', data='删除成功')

api/api_v1/endpoints/project.py

@@ -0,0 +1,369 @@
+import pymongo
+from bson import ObjectId
+from fastapi import APIRouter, Depends, Request
+from motor.motor_asyncio import AsyncIOMotorDatabase
+import crud, schemas
+from api import deps
+from core.config import settings
+from db import get_database
+from db.ckdb import CKDrive, get_ck_db
+from schemas.project import ProjectCreate
+# from utils import casbin_enforcer
+from utils import casbin_enforcer
+
+router = APIRouter()
+
+
+@router.post("/create")
+async def create(
+        request: Request,
+        data_in: ProjectCreate,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """创建项目"""
+    try:
+        res_project = await crud.project.create(db, data_in, current_user=request.user)
+        await crud.project_number.createxiangmu(db, data_in)
+        # 同步插入项目
+        # await crud.project_number.createxiangmu(db, data_in)
+        # 同步存入root权限中新项目的权限
+        user_url = await crud.user_url.get_quanxian(db,
+                                                    schemas.Url_quanxian(user_id='04491842be9811eb8acdd5bd867f57d6'))
+        user_url['game'].append(data_in.game)
+        user_url['quanxian_id'].append('ab1')
+        user_url['quanxian'].append('root')
+        await crud.user_url.updata_quanxian(db, schemas.Url_quanxian(user=user_url['user'], user_id=user_url['user_id'],
+                                                                     game=user_url['game'],
+                                                                     quanxian_id=user_url['quanxian_id'],
+                                                                     quanxian=user_url['quanxian']))
+    except pymongo.errors.DuplicateKeyError:
+        return schemas.Msg(code=-1, msg='项目名已存在', data='项目名已存在')
+
+    folder = schemas.FolderCreate(
+        name='未分组',
+        project_id=res_project.inserted_id,
+        cat='kanban',
+        pid=res_project.inserted_id,
+    )
+    await crud.folder.create(db, folder, user_id=current_user.id)
+    folder = schemas.FolderCreate(
+        name='共享给我的',
+        project_id=res_project.inserted_id,
+        cat='kanban',
+        pid=res_project.inserted_id,
+    )
+    await crud.folder.create(db, folder, user_id=current_user.id)
+
+    # # 创建全部数据权限
+    # data_auth = schemas.DataAuthCreate(name='全部', data=['*'])
+    # await crud.data_auth.create(db, data_auth, data_in.game)
+
+    # 新建项目管理员权限
+    # role_name = f'{data_in.game}_admin'
+    # role_dom = data_in.game
+    # casbin_enforcer.add_policy(role_name, role_dom, '*', '*')
+    # await crud.authority.create(db, 'p', role_name, role_dom, '*', '*')
+    # 添加角色
+    # await crud.authority.create(db, 'g', settings.SUPERUSER_NAME, role_name, '*', '*', role_name='系统项目管理员', game='*')
+    # # 添加数据权限
+    # await crud.authority.set_data_auth(db, schemas.DataAuthSet(username=request.user.username, data_auth_id='*'),
+    #                                    game=data_in.game, v1=role_name)
+    return schemas.Msg(code=0, msg='创建成功')
+
+
+@router.get("/")
+async def read_project(request: Request,
+                       db: AsyncIOMotorDatabase = Depends(get_database),
+                       current_user: schemas.UserDB = Depends(deps.get_current_user)
+                       ):
+    """查看自己拥有的项目"""
+    if request.user.username == 'root':
+        resp = await crud.project.all_game(db)
+        resp = sorted(resp, key=lambda x: x.get('sort') or 999)
+    else:
+        # game_list = casbin_enforcer.get_domains_for_user(request.user.username)
+        # resp = await crud.project.get_my_game(db, game_list)
+        project_data = await crud.user_url.get_quanxian(db, schemas.Url_quanxian(user_id=request.user.id))
+        resp = await crud.project.get_my_game(db, project_data['game'])
+    return schemas.Msg(code=0, msg='ok', data=resp)
+#获取项目名和渠道名project_name
+@router.get("/project_name")
+async def project_name(request: Request,
+                       db: AsyncIOMotorDatabase = Depends(get_database),
+                       current_user: schemas.UserDB = Depends(deps.get_current_user)
+                       ):
+    if request.user.username == 'root':
+        res = await crud.project_number.all_xiangmu(db)
+        for i in res:
+            i['_id'] = str(i['_id'])
+    return schemas.Msg(code=0,msg='ok',data=res)
+#添加项目名，渠道名
+@router.post("/add_project_name")
+async def add_project_name(request: Request,
+        data_in: schemas.ProjectnumberInsert,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)):
+    #插入数据
+    #await crud.project_number.create(db, data_in)
+    #修改数据
+    await crud.project_number.update(db, data_in)
+    return schemas.Msg(code=0, msg='修改成功', data=True)
+@router.get("/detail")
+async def detail(request: Request,
+                 game: str,
+                 db: AsyncIOMotorDatabase = Depends(get_database),
+                 ck: CKDrive = Depends(get_ck_db),
+                 current_user: schemas.UserDB = Depends(deps.get_current_user)
+                 ):
+    """查看项目信息"""
+    res = await crud.project.find_one(db, {'game': game})
+    event_count = await ck.count(game, 'event')
+    user_count = await ck.count(game, 'user_view')
+    event_type_count = await ck.distinct_count(game, 'event', '#event_name')
+    event_attr_count = await ck.field_count(db=game, tb='event')
+    user_attr_count = await ck.field_count(db=game, tb='user_view')
+
+    res['event_count'] = event_count
+    res['user_count'] = user_count
+    res['event_type_count'] = event_type_count
+    res['event_attr_count'] = event_attr_count
+    res['user_attr_count'] = user_attr_count
+    return schemas.Msg(code=0, msg='ok', data=res)
+
+
+@router.post("/rename")
+async def rename_project(request: Request,
+                         game: str,
+                         data_in: schemas.ProjectRename,
+                         db: AsyncIOMotorDatabase = Depends(get_database),
+                         current_user: schemas.UserDB = Depends(deps.get_current_user)
+                         ):
+    """修改项目名"""
+    try:
+        res = await crud.project.rename(db, data_in)
+    except pymongo.errors.DuplicateKeyError:
+        return schemas.Msg(code=-1, msg='项目名已存在')
+    return schemas.Msg(code=0, msg='ok', data=res)
+
+
+@router.post("/add_members")
+async def add_members(request: Request,
+                      game: str,
+                      data_in: schemas.ProjectAddMember,
+                      db: AsyncIOMotorDatabase = Depends(get_database),
+                      current_user: schemas.UserDB = Depends(deps.get_current_user)
+                      ):
+    """项目添加成员"""
+
+    for item in data_in.members:
+        # casbin_enforcer.add_grouping_policy(item.username, item.role_name, game)
+        # # 设置数据权限
+        # await crud.authority.set_data_auth(db,
+        #                                    schemas.DataAuthSet(username=item.username, data_auth_id=item.data_auth_id),
+        #                                    game)
+
+        folder = schemas.FolderCreate(
+            name='未分组',
+            project_id=data_in.project_id,
+            cat='kanban',
+            pid=data_in.project_id,
+        )
+        await crud.folder.create(db, folder, user_id=item.user_id)
+
+    await crud.project.add_members(db, schemas.ProjectMember(project_id=data_in.project_id,
+                                                             members=[item.username for item in data_in.members]))
+
+    return schemas.Msg(code=0, msg='ok', data=data_in)
+@router.post("/import_member")
+async def import_member(request: Request,
+                      game:str,
+                      data_in:schemas.Import_project,
+                      db: AsyncIOMotorDatabase = Depends(get_database),
+                      current_user: schemas.UserDB = Depends(deps.get_current_user)
+                      ):
+    """成员管理中的导入其他项目的成员到本项目中"""
+    res=await crud.user_url.get_all(db)
+    for i in res:
+        for nu in range(len(i['game'])):
+            if data_in.games == i['game'][nu] and game not in i['game']:
+                i['game'].append(game)
+                i['quanxian_id'].append(i['quanxian_id'][nu])
+                i['quanxian'].append(i['quanxian'][nu])
+                await crud.user_url.updata_quanxian(db,schemas.Url_quanxian(game=i['game'],user=i['user'],user_id=i['user_id'],quanxian_id=i['quanxian_id'],
+                                                                            quanxian=i['quanxian']))
+
+    return schemas.Msg(code=0, msg='ok',data='' )
+
+@router.post("/edit_member")
+async def edit_member(request: Request,
+                      game: str,
+                      db: AsyncIOMotorDatabase = Depends(get_database),
+                      current_user: schemas.UserDB = Depends(deps.get_current_user)
+                      ):
+    """编辑成员权限  角色和数据"""
+
+    pass
+    return schemas.Msg(code=0, msg='ok', )
+
+
+#
+#
+@router.get("/members")
+async def members(request: Request,
+                  game: str,
+                  db: AsyncIOMotorDatabase = Depends(get_database),
+                  current_user: schemas.UserDB = Depends(deps.get_current_user)
+                  ):
+    """查看项目成员"""
+    # data = casbin_enforcer.get_all_users_by_domain(game)
+    # names = []
+    # role_ids = []
+    # for item in data:
+    #     names.append(item['username'])
+    #     role_ids.append(item['role_id'])
+    # users = await crud.user.get_by_users(db, {'name': {'$in': names}})
+    # roles = await crud.role.find_ids(db, role_ids)
+    # users = {item.name: item.dict() for item in users.data}
+    # roles = {item['_id']: item['name'] for item in roles}
+    # res = []
+    # for item in data:
+    #     username = item['username']
+    #     role_id = item['role_id']
+    #     try:
+    #         res.append({
+    #             **users[username],
+    #             'role': roles[role_id],
+    #             'role_id': role_id,
+    #         })
+    #     except:
+    #         pass
+    #     # res.append({
+    #     #         **users[username],
+    #     #         'role': roles[role_id],
+    #     #         'role_id': role_id,
+    #     #     })
+    # return schemas.Msg(code=0, msg='ok', data=res)
+    res = await crud.user_url.get_all(db)
+    # 符合当前项目权限的用户
+    names = []
+    # 符合当前项目权限的用户的对应权限级别
+    quanxian = {}
+    quanxian_id = {}
+    for i in res:
+        for nu in range(len(i['game'])):
+            if game == i['game'][nu]:
+                names.append(i['user'])
+                quanxian[i['user']] = i['quanxian'][nu]
+                quanxian_id[i['user']] = i['quanxian_id'][nu]
+    users = await crud.user.get_by_users(db, {'name': {'$in': names}})
+    data = []
+    for use in users.data:
+        use = use.dict()
+        use['role'] = quanxian[use['name']]
+        use['role_id'] = quanxian[use['name']]
+        data.append(use)
+    return schemas.Msg(code=0, msg='ok', data=data)
+# @router.post("/del_member")
+# async def members(request: Request,
+#                   game: str,
+#                   data_in: schemas.ProjectDelMember,
+#                   db: AsyncIOMotorDatabase = Depends(get_database),
+#                   current_user: schemas.UserDB = Depends(deps.get_current_user)
+#                   ):
+#     """删除项目成员"""
+#     # casbin_enforcer.delete_roles_for_user_in_domain(data_in.username, data_in.role, game)
+#     await crud.project.del_members(db, data_in)
+#     # await crud.authority.delete(db, ptype='g', v2=game, v0=data_in.username)
+#     return schemas.Msg(code=0, msg='ok')
+
+
+@router.post("/clean")
+async def read_kanban(
+        data_in: schemas.ProjectClean,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+):
+    """
+    清理项目  删除项目所有内容
+    :param data_in:
+    :param db:
+    :param current_user:
+    :return:
+    """
+    # 删除 报表
+    await crud.report.delete(db, {'project_id': data_in.project_id})
+    # 删除 空间
+    await crud.space.delete(db, {'project_id': data_in.project_id})
+    # 删除 文件夹
+    await crud.folder.delete(db, {'project_id': data_in.project_id})
+    # 删除 看板
+    await crud.dashboard.delete(db, {'project_id': data_in.project_id})
+
+    return schemas.Msg(code=0, msg='ok')
+
+
+@router.post("/kanban")
+async def read_kanban(
+        data_in: schemas.ProjectKanban,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+):
+    """获取自己的看板"""
+    res = {'kanban': [], 'spaces': []}
+    # 我的看板
+    kanban = await crud.folder.read_folder(db, project_id=data_in.id, user_id=current_user.id, cat='kanban')
+
+    for item in kanban:
+        res['kanban'].append({
+            'name': item['name'],
+            'children': [],
+            '_id': item['_id']
+        })
+        async for d in crud.dashboard.find(db, {'pid': item['_id']}).sort([('sort', 1)]):
+            res['kanban'][-1]['children'].append({
+                'name': d['name'],
+                '_id': d['_id']
+            })
+
+    # 我的空间
+    where = {
+        'project_id': data_in.id,
+        'members.user_id': current_user.id
+        # '$or': [{'rw_members': current_user.id}, {'r_members': current_user.id}]
+    }
+    spaces = await crud.space.find_many(db, where)
+    # 空间 文件夹  看板
+    for item in spaces:
+        res['spaces'].append({
+            'name': item['name'],
+            'children': [],
+            '_id': item['_id']
+        })
+        authority = {item['user_id']: item['authority'] for item in item['members']}
+        res['spaces'][-1]['authority'] = authority.get(current_user.id, 'r')
+
+        for f in await crud.folder.find_many(db, {'pid': item['_id']}):
+            res['spaces'][-1]['children'].append({
+                'name': f['name'],
+                '_id': f['_id'],
+                'children': [],
+                'isFolder': True
+            })
+
+            async for d in crud.dashboard.find(db, {'pid': item['_id']}).sort([('sort', 1)]):
+                res['spaces'][-1]['children'][-1]['children'].append({
+                    'name': d['name'],
+                    '_id': d['_id']
+                })
+
+        # 空间 看板
+        async for d in crud.dashboard.find(db, {'pid': item['_id']}).sort([('sort', 1)]):
+            res['spaces'][-1]['children'].append({
+                'name': d['name'],
+                '_id': d['_id'],
+                'user_id':d['user_id'],
+                'isFolder': False
+            })
+
+    return schemas.Msg(code=0, msg='ok', data=res)

api/api_v1/endpoints/report.py

@@ -0,0 +1,166 @@
+from typing import Any
+
+import pymongo
+from fastapi import APIRouter, Depends, Request
+from motor.motor_asyncio import AsyncIOMotorDatabase
+import crud, schemas
+
+from db import get_database
+from api import deps
+from utils import get_uid
+
+router = APIRouter()
+
+
+@router.post("/create")
+async def create(
+        request: Request,
+        data_in: schemas.ReportCreate,
+        game: str,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """新建报表"""
+    try:
+        await crud.report.create(db, data_in, user_id=request.user.id)
+    except pymongo.errors.DuplicateKeyError:
+        return schemas.Msg(code=-1, msg='error', data='报表已存在')
+
+    return schemas.Msg(code=0, msg='ok', data='创建成功')
+
+
+@router.post("/edit")
+async def edit(
+        request: Request,
+        data_in: schemas.ReportEdit,
+        game: str,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """编辑报表"""
+    res = await crud.report.update_one(db, {'_id': data_in.report_id, 'user_id': request.user.id},
+                                       {'$set': {'query': data_in.query, 'name': data_in.name, 'desc': data_in.desc}})
+    # if not res.matched_count:
+    # #if res.matched_count:
+    #     return schemas.Msg(code=-1, msg='只能报表所有者编辑')
+    return schemas.Msg(code=0, msg='ok', data='编辑成功')
+
+
+@router.post("/copy")
+async def copy(
+        request: Request,
+        data_in: schemas.ReportCopy,
+        game: str,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """复制报表到其他项目"""
+
+    for report_id in data_in.report_ids:
+        new_report = await crud.report.get(db, report_id)
+        if not new_report:
+            continue
+        new_report_id = get_uid()
+        new_report['_id'] = new_report_id
+        new_report['project_id'] = data_in.dest_project_id
+        await crud.report.insert_one(db, new_report)
+    return schemas.Msg(code=0, msg='ok', data='编辑成功')
+
+
+@router.post("/read_report")
+async def read_report(
+        request: Request,
+        data_in: schemas.ReportRead,
+        game: str,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> Any:
+    """获取已建报表 属于自己的"""
+    ext_where = dict()
+    dashboard = dict()
+    if data_in.report_id:
+        ext_where = {'_id': {'$in': data_in.report_id}}
+    else:
+        ext_where['user_id'] = request.user.id
+    if data_in.dashboard_id:
+        dashboard = await crud.dashboard.get(db, id=data_in.dashboard_id)
+    # projection = {'query': False}
+    projection = None
+    reports = await crud.report.read_report(db, project_id=data_in.project_id,
+                                            projection=projection, **ext_where)
+
+    for item in reports:
+        item['added'] = False
+        # item['name'] = item['name']
+        item['show_config'] = dict()
+        added_ids = {item['report_id']: item for item in dashboard.get('reports', [])}
+        if item['_id'] in added_ids:
+            item['added'] = True
+            item['show_config'] = added_ids[item['_id']]
+        #保存的看板按备注显示的数据显示
+        if type(item['query']['events']) == list:
+            event_show_name = await crud.event_mana.get_all_show_name(db, game)
+            for i in item['query']['events']:
+                if 'event_name' in i:
+                    if i['event_name'] in event_show_name:
+                        if 'event_desc' in i :
+                                event_name= i['event_name']
+                                i['event_desc']= event_show_name[event_name]
+                        else:
+                            event_name = i['event_name']
+                            i['eventDesc'] = event_show_name[event_name]
+                else:
+                    if i['eventName'] in event_show_name:
+                        if 'event_desc' in i :
+                                event_name= i['eventName']
+                                i['event_desc']= event_show_name[event_name]
+                        else:
+                            event_name = i['eventName']
+                            i['eventDesc'] = event_show_name[event_name]
+#放置争霸
+        if type(item['query']['events']) == dict:
+            data_attr = await crud.data_attr.find_many(db, {'game': game})
+            data_attr = {item['name']: item for item in data_attr}
+            item_dict=item['query']['events']
+            for k,v in item_dict.items():
+                if k == 'quotaDesc':
+                    if item_dict['quotaDesc'] in data_attr:
+                        item_dict['quotaDesc']=data_attr[item_dict['quotaDesc']]['show_name']
+
+
+
+
+
+        # for k,v in event_show_name.items():
+        #     if 'event_desc' in item['query']['events'][0]:
+        #         event_desc = item['query']['events'][0]['event_desc']
+        #         if k == event_desc:
+        #             item['query']['events'][0]['event_desc'] = event_show_name[event_desc]
+        #     else:
+        #         event_desc = item['query']['events'][0]['eventDesc']
+        #         if k == event_desc:
+        #             item['query']['events'][0]['eventDesc'] = event_show_name[event_desc]
+
+    reports = sorted(reports, key=lambda x: x.get('show_config', {'sort': 999}).get('sort', 999) or 999)
+
+    return schemas.Msg(code=0, msg='ok', data=reports)
+
+
+@router.post("/delete")
+async def delete(
+        request: Request,
+        data_in: schemas.ReportDelete,
+        game: str,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """删除报表"""
+    # 删除Report 自己创建的
+    del_report = await crud.report.delete(db, {'_id': data_in.id, 'user_id': current_user.id})
+    # 从看板弹出
+    del_item = {'report_id': data_in.id}
+    await crud.dashboard.update_many(db, {}, {'$pull': {'reports': del_item}})
+
+    if del_report.deleted_count == 0:
+        return schemas.Msg(code=-1, msg='error', data='删除失败')
+    return schemas.Msg(code=0, msg='ok', data='删除成功')

api/api_v1/endpoints/space.py

@@ -0,0 +1,99 @@
+import pymongo
+from fastapi import APIRouter, Depends
+from motor.motor_asyncio import AsyncIOMotorDatabase
+import crud, schemas
+
+from db import get_database
+from api import deps
+
+router = APIRouter()
+
+
+@router.post("/create")
+async def create(
+        game: str,
+        data_in: schemas.SpaceCreate,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """创建空间"""
+    try:
+        if data_in.is_all_member:
+            data_in.members.clear()
+            users = await crud.user.find_many(db)
+            for user in users:
+                if user['_id'] == current_user.id:
+                    continue
+                data_in.members.append(schemas.space.Member(**user, authority=data_in.authority))
+        await crud.space.create(db, data_in, user=current_user)
+    except pymongo.errors.DuplicateKeyError:
+        return schemas.Msg(code=-1, msg='空间已存在', data='空间已存在')
+
+    return schemas.Msg(code=0, msg='创建成功', data='创建成功')
+
+
+@router.post("/delete")
+async def delete(
+        data_in: schemas.SpaceDelete,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """删除空间"""
+    # 删除空间 自己创建的
+    del_space = await crud.space.delete(db, {'_id': data_in.id})
+    # 删除文件夹
+    del_folder = await crud.folder.find_many(db, {'pid': data_in.id})
+    del_folder_ids = [f['_id'] for f in del_folder]
+    await crud.folder.delete(db, {'pid': data_in.id})
+    # 删除文件夹下的 dashboard
+    if del_folder_ids:
+        await crud.dashboard.delete(db, {'_id': {'$in': del_folder_ids}})
+
+    # 删除空间下的 dashboard
+    await crud.dashboard.delete(db, {'pid': data_in.id})
+
+    if del_space.deleted_count == 0:
+        return schemas.Msg(code=-1, msg='error', data='删除失败')
+    return schemas.Msg(code=0, msg='ok', data='删除成功')
+
+
+@router.post("/rename")
+async def rename(
+        data_in: schemas.SpaceRename,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """重命名空间"""
+    res = await crud.space.rename(db, data_in)
+
+    return schemas.Msg(code=0, msg='ok', data=1)
+
+
+@router.post("/set_members")
+async def set_members(
+        data_in: schemas.AddSpaceMembers,
+        game: str,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """设置空间成员"""
+    res = await crud.space.set_members(db, data_in)
+    return schemas.Msg(code=0, msg='ok', data=1)
+
+
+@router.post("/detail")
+async def detail(
+        data_in: schemas.SpaceDetail,
+        game: str,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """空间详细"""
+    space_info = await crud.space.get(db, id=data_in.space_id)
+    exists_member = {item.get('user_id') for item in space_info.get('members', [])}
+    members_info = await crud.user.find_ids(db, list(exists_member))
+    members_info = {item['_id']: item for item in members_info}
+    for item in space_info.get('members', []):
+        if user_info := members_info.get(item['user_id']):
+            item.update(schemas.UserDB(**user_info).dict(by_alias=True))
+    return schemas.Msg(code=0, msg='ok', data=space_info)

api/api_v1/endpoints/test.py

@@ -0,0 +1,39 @@
+from typing import Any
+
+from fastapi import APIRouter, Depends, Request
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import crud
+import schemas
+from api import deps
+from db import get_database
+from db.ckdb import CKDrive, get_ck_db
+from db.redisdb import RedisDrive, get_redis_pool
+from models.behavior_analysis import BehaviorAnalysis
+
+router = APIRouter()
+
+
+@router.post("/test")
+async def test(
+        request: Request,
+        rdb: RedisDrive = Depends(get_redis_pool),
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.UserDB = Depends(deps.get_current_user),
+        ):
+
+        """api 列表"""
+        app = request.app
+        data = {}
+        for r in app.routes:
+            title = r.tags[0] if hasattr(r, 'description') else None
+            if not title:
+                continue
+            data.setdefault(title, {'list': []})
+            path = r.path
+            name = r.description if hasattr(r, 'description') else r.name
+            data[title]['list'].append({'api': path, 'title': name})
+
+        res = [{'title': k, 'list': v['list']} for k, v in data.items()]
+
+        return schemas.Msg(code=0, msg='ok', data=res)

api/api_v1/endpoints/user.py

@@ -0,0 +1,161 @@
+from datetime import timedelta
+from typing import Any
+
+from fastapi import APIRouter, Body, Depends, HTTPException, Request
+from fastapi.security import OAuth2PasswordRequestForm
+from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorDatabase
+
+import crud, schemas
+from api import deps
+from core import security
+from core.config import settings
+from utils import get_uid
+from db import get_database
+
+router = APIRouter()
+
+
+@router.post("/login")
+async def login(
+        # data: schemas.UserLogin,
+        data: OAuth2PasswordRequestForm = Depends(),
+        db: AsyncIOMotorDatabase = Depends(get_database)
+) -> Any:
+    """
+    OAuth2兼容令牌登录，获取将来令牌的访问令牌
+    """
+    user = await crud.user.authenticate(db,
+                                        name=data.username, password=data.password
+                                        )
+    if not user:
+        # raise HTTPException(status_code=400, detail="Incorrect name or password")
+        return schemas.Msg(code=-1, msg='密码或用户名错误')
+    access_token_expires = timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
+    # access_token_expires = timedelta(seconds=5)
+    await crud.user.update_login_time(db, data.username)
+
+    return {
+        'data': {
+            'name': user.name,
+            'nickname': user.nickname,
+            'email': user.email,
+            'tel': user.tel,
+            'userid':user.id,
+
+            'token': security.create_access_token(
+                expires_delta=access_token_expires, _id=str(user.id), email=user.email,
+                nickname=user.nickname,
+                is_superuser=user.is_superuser, name=user.name,
+                data_where=user.data_where,
+            ),
+            "token_type": "bearer",
+
+        },
+        'access_token': security.create_access_token(
+            expires_delta=access_token_expires, _id=str(user.id), email=user.email,
+            nickname=user.nickname,
+            is_superuser=user.is_superuser, name=user.name, data_where=user.data_where
+        ),
+        "token_type": "bearer",
+
+        'code': 0,
+        'msg': 'success',
+    }
+
+
+@router.get("/me", response_model=schemas.User)
+def me(current_user: schemas.User = Depends(deps.get_current_user)) -> Any:
+    """
+    Test access token
+    """
+    return current_user
+
+
+@router.post("/reset_password")
+async def reset_password(request: Request,
+                         game: str,
+                         data_in: schemas.UserRestPassword,
+                         db: AsyncIOMotorDatabase = Depends(get_database),
+                         current_user: schemas.User = Depends(deps.get_current_user)
+                         ) -> Any:
+    """
+    修改其他人密码
+    """
+    try:
+        await crud.user.reset_password(db, data_in)
+    except Exception as e:
+        return schemas.Msg(code=0, msg='修改失败', data={'username': data_in})
+    return schemas.Msg(code=0, msg='ok')
+
+
+@router.post("/reset_my_password")
+async def reset_password(request: Request,
+                         game: str,
+                         data_in: schemas.UserRestMyPassword,
+                         db: AsyncIOMotorDatabase = Depends(get_database),
+                         current_user: schemas.User = Depends(deps.get_current_user)
+                         ) -> Any:
+    """
+    修改自己的密码
+    """
+    await crud.user.reset_password(db, schemas.UserRestPassword(username=current_user.name, password=data_in.password))
+    return schemas.Msg(code=0, msg='ok')
+
+
+@router.post("/edit_profile")
+async def edit_profile(request: Request,
+                       game: str,
+                       data_in: schemas.UserProfileEdit,
+                       db: AsyncIOMotorDatabase = Depends(get_database),
+                       current_user: schemas.User = Depends(deps.get_current_user)
+                       ) -> Any:
+    """
+    编辑用户资料
+    """
+    await crud.user.edit_profile(db, data_in, user_id=request.user.id)
+    return schemas.Msg(code=0, msg='ok', data=data_in)
+
+
+@router.get("/all_account")
+async def all_account(page: int = 1, limit: int = 100, db: AsyncIOMotorDatabase = Depends(get_database),
+                      current_user: schemas.User = Depends(deps.get_current_user)
+                      ) -> Any:
+    """
+    获取所有用户
+    """
+    page -= 1
+    if page < 0:
+        page = 0
+    cursor = crud.user.find(db).skip(page * limit).limit(limit)
+
+    data = [schemas.UserDB(**user) async for user in cursor]
+
+    return schemas.Msg(code=0, msg='ok', data=data)
+
+
+@router.post("/add_account")
+async def all_account(
+        data_in: schemas.CreateAccount,
+        db: AsyncIOMotorDatabase = Depends(get_database),
+        current_user: schemas.User = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """
+    创建新账号
+    """
+    created = []
+    id = []
+    for name in data_in.account_list:
+        if is_exists := await crud.user.exists(db, {'name': name}):
+            continue
+        else:
+            new_account = schemas.UserCreate(name=name, password='123')
+            created.append(name)
+            #创建账户并返回id
+            id_one=await crud.user.create(db, new_account)
+            id.append(id_one)
+    res = {
+        'created_account': created,
+        'password': '123',
+        'id':id
+    }
+    return schemas.Msg(code=0, msg='ok', data=res)

api/api_v1/endpoints/xquery.py

@@ -0,0 +1,271 @@
+import datetime
+import mimetypes
+from collections import defaultdict
+import time
+from urllib.parse import quote
+import re
+from clickhouse_driver import Client
+import pandas as pd
+import numpy as np
+from fastapi import APIRouter, Depends, Request
+from motor.motor_asyncio import AsyncIOMotorDatabase
+from pandas import DataFrame
+from starlette.responses import StreamingResponse
+
+import crud, schemas
+from common import *
+
+from api import deps
+from db import get_database
+from db.ckdb import get_ck_db, CKDrive, ckdb
+from db.redisdb import get_redis_pool, RedisDrive
+
+from models.behavior_analysis import BehaviorAnalysis
+from models.user_analysis import UserAnalysis
+from models.x_analysis import XAnalysis
+from utils import DfToStream, get_bijiao
+
+router = APIRouter()
+
+
+@router.post("/ltv_model_sql")
+async def ltv_model_sql(
+        request: Request,
+        game: str,
+        analysis: XAnalysis = Depends(XAnalysis),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """ ltv模型sql """
+    await analysis.init(data_where=current_user.data_where)
+    data = analysis.ltv_model_sql()
+    return schemas.Msg(code=0, msg='ok', data=[data])
+
+
+@router.post("/ltv_model")
+async def ltv_model_sql(
+        request: Request,
+        game: str,
+        analysis: XAnalysis = Depends(XAnalysis),
+        ckdb: CKDrive = Depends(get_ck_db),
+        current_user: schemas.UserDB = Depends(deps.get_current_user)
+) -> schemas.Msg:
+    """ ltv模型sql """
+    await analysis.init(data_where=current_user.data_where)
+    res = analysis.ltv_model_sql()
+    sql = res['sql']
+    #仅一条筛选条件则是把GM过滤后获取全部数据
+    if len(analysis.global_filters)==1 and analysis.global_filters[0]['strftv']=='GM':
+        try:
+            df = await ckdb.query_dataframe(sql)
+        except Exception as e:
+            return schemas.Msg(code=-9, msg='报表配置参数异常')
+    #多条筛选条件则合成新的sql
+    else:
+        new_sql=""""""
+        #拆分sql
+        split_sql = sql.split('AND 1')
+        #获取每一条筛选条件
+        for i in analysis.global_filters:
+            #剔除GM
+            if i['strftv'] != 'GM':
+                #获取筛选条件的包含关系
+                bijiao=get_bijiao(i["comparator"])
+                #获取筛选条件的值
+                condition=tuple(i['ftv'])
+                #获取事件名
+                columnName=i['columnName']
+                dd = f""" AND {game}.event.{columnName} {bijiao} {condition}"""
+                new_sql+=dd
+        split_="""AND 1 """
+        news_sql = split_sql[0] + split_+new_sql + split_sql[1] + split_+new_sql+ split_sql[2]+split_+split_sql[3]
+        df = await ckdb.query_dataframe(news_sql)
+    # # 判断11月23号之前的数据
+    # list_data_range=analysis.date_range
+    # liststr_data_range=[]
+    # for i in list_data_range:
+    #     liststr_data_range.append(str(i))
+    # quota = analysis.event_view['quota']
+    # #判断是设备LTV则执行下面代码，如是角色实充LTV则不执行
+    # if quota == '#distinct_id':
+    #     if '2021-11-22' in liststr_data_range or '2021-11-22' >=liststr_data_range[-1]:
+    #         #取搜索最后为11.23号之前的数据
+    #         if '2021-11-22' >=liststr_data_range[-1]:
+    #             news_sql=""""""
+    #             split_sql=sql.split('AND is_new_device = 1')
+    #             new_sql=split_sql[0]+split_sql[1]+split_sql[2]
+    #             news_sql+=new_sql
+    #             df_twenty_three=await ckdb.query_dataframe(news_sql)
+    #         #取包含有11.23号之前和23号之后的那一段
+    #         else:
+    #             start_date=str(list_data_range[0])
+    #             end_date='2021-11-22'
+    #             news_sql = """"""
+    #             split_sql = sql.split('AND is_new_device = 1')
+    #             for i in split_sql:
+    #                 news_sql += i
+    #             #用正则表达式切时间
+    #             zhengze_time=r'\d{4}-\d{1,2}-\d{1,2}'
+    #             zhengze_sql=re.split(zhengze_time,news_sql)
+    #             zz_new_sql=zhengze_sql[0]+start_date+zhengze_sql[1]+end_date+zhengze_sql[2]+start_date+zhengze_sql[3]+end_date+zhengze_sql[4]
+    #             zz_news_sql=""""""
+    #             zz_news_sql+=zz_new_sql
+    #             df_twenty_three = await ckdb.query_dataframe(zz_news_sql)
+    #         #上下合并两组数据，忽略以前的索引下标
+    #         df= pd.concat([df,df_twenty_three], axis=0, ignore_index=True)
+    #         df.sort_values('date', inplace=True)
+    #         #去重
+    #         #df.drop_duplicates(inplace=True)
+    quota = res['quota']  #字段名
+    ltv_n = res['ltv_n']
+    #df = await ckdb.query_dataframe(sql)
+    if df.empty:
+        return schemas.Msg(code=-9, msg='查无数据')
+    df.fillna(0, inplace=True)   #修改原对象，以0填补空缺值
+
+    # for d in set(res['date_range']) - set(df['date']):  # 时间的差集运算 最后为空
+    #     df.loc[len(df)] = 0
+    #     df.loc[len(df) - 1, 'date'] = d
+    #     days = (pd.Timestamp.now().date() - d).days  # 时间差
+    #     # if days + 2 >= ltv_len:
+    #     #     continue
+    #     df.iloc[len(df) - 1, days + 3:] = '-'
+    # df.sort_values('date', inplace=True)  # 根据date进行倒叙排序
+
+    for d in set(res['date_range']) - set(df['date']):
+        #在有效日期最后一行补充行数据（值都为'-'），补充的行数为两个集合的差集长度
+        df.loc[len(df)] = '-'
+        #在date此列补充多行数据（值为两个集合差集的子元素）
+        df.loc[len(df) - 1, 'date'] = d
+        # days = (d-pd.Timestamp.now().date()).days
+        # # if days + 2 >= ltv_len:
+        # #     continue
+        # if days>0:
+        #     df.iloc[len(df) - 1, 1:] = '-'
+
+    df.sort_values('date', inplace=True)
+    df.rename(columns={'date': '注册日期'}, inplace=True)   #True为将结果返回赋值给原变量，修改原对象，columns为列名
+    cat = '角色数'
+    if quota == '#distinct_id':  #如果字段名=字段名
+        cat = '设备数'
+    df.rename(columns={'cnt1': cat}, inplace=True)   #原数据基础上修改df里面列名为cnt1为设备数
+    df1 = df[['注册日期', cat, *[f'LTV{i}' for i in ltv_n]]]   #1, 2, 3, 4, 5, 6, 7, 8, 9, ~~到360
+    df2 = df[['注册日期', cat, *[f'sumpay_{i}' for i in ltv_n]]]
+    df2.replace('-', 0, inplace=True)  #True改变原数据，前面是需要替换的值，后面是替换后的值。 在原数据把下划线替换成0
+    #修改下面代码
+    # 去除sumpay_2的值为0的列
+    new_df2 = (df2.drop(df2[(df2.sumpay_2 == 0)].index))
+    #为new_df2排序
+    new_df2=new_df2.reset_index(drop=True)
+    #求相差天数
+    str_time = df2['注册日期'][0]
+    #str_time =new_df2['注册日期'][0]
+    str_time01=str(str_time)
+    split_time = str_time01.split('-')
+    #str_time = str(res['date_range'][0])
+    # split_time = str_time.split('-')
+    now_time=time.strftime("%Y-%m-%d",time.localtime())
+    split_now_time = now_time.split('-')
+    today = datetime.datetime(int(split_time[0]), int(split_time[1]), int(split_time[2]))
+    now_day = datetime.datetime(int(split_now_time[0]), int(split_now_time[1]), int(split_now_time[2]))
+    newday = (now_day - today).days + 1
+    #计算方法运算每个LTV的均值
+    _listData = {}
+    for i in ltv_n:
+        if i <=newday:
+            #计算均值
+            #avgLtv = (new_df2[[f'sumpay_{i}']][0:newday + 1 - i].sum() / new_df2[cat][0:newday + 1 - i].sum()).round(2)
+            #12.20号计算LTV均值的时候分母包括当天未充值新增设备数，比剔除掉的计算值偏小
+            avgLtv = (df2[[f'sumpay_{i}']][0:newday + 1 - i].sum() / df2[cat][0:newday + 1 - i].sum()).round(2)
+            #取出均值
+            new_avgLtv=str(avgLtv).split('\n')[0].split(' ')
+            new_avgLtv01=new_avgLtv[len(new_avgLtv)-1]
+            if new_avgLtv01 == 'NaN':
+                _listData[f'sumpay_{i}'] = '-'
+            else:
+                _listData[f'sumpay_{i}'] = new_avgLtv01
+
+            #原代码
+            # avgLtv=(df2[[f'sumpay_{i}']][0:newday+1-i].sum()/df2[cat][0:newday+1-i].sum()).round(2)
+            # new_avgLtv=str(avgLtv).split('\n')[0].split(' ')
+            # new_avgLtv01=new_avgLtv[len(new_avgLtv)-1]
+            # if new_avgLtv01 == 'NaN':
+            #     _listData[f'sumpay_{i}'] = '-'
+            # else:
+            #     _listData[f'sumpay_{i}'] = new_avgLtv01
+        else:
+            _listData[f'sumpay_{i}']='-'
+    avgLtvlist = pd.Series(_listData)
+
+    _listname=[]
+    #计算总累计LTV最后一个值
+    for k, v in _listData.items():
+        if v != 0 or v!= '-':
+        # if v !=0:
+            _listname.append(k)
+    max_nmu=max(_listname)
+    #max_num = (new_df2[[max_nmu]].sum() / new_df2[cat].sum()).round(2)
+    max_num=(df2[[max_nmu]].sum()/df2[cat].sum()).round(2)
+    max_number=str(max_num[0])
+    df1.loc[len(df1)] = ['均值', df2[cat].sum(), *avgLtvlist]
+    #原代码
+    #df1.loc[len(df1)] = ['均值', df2[cat].sum(), *avgLtvlist]
+
+    # avg_ltv = (df2[[f'sumpay_{i}' for i in ltv_n]].sum() / df2[cat].sum()).round(2)
+    #df1.loc[len(df1)] = ['均值', df2[cat].sum(), *avg_ltv]
+    df1.insert(2, '累计LTV', 0)
+    last_ltv = []
+    for items in df1.values:
+        for item in items[::-1]:
+            if item != '-':
+                last_ltv.append(item)
+                break
+    #修改累计LTV中最后一个值
+    last_ltv[-1]=max_number
+
+
+    df1['累计LTV'] = last_ltv
+
+
+    #把列中累计LTV等于0的值改成'-'
+    #df1.loc[df1['累计LTV']==0, '累计LTV'] = '-'
+    #剔除行，列的累计LTV=='-'的剔除出去
+    df3 = df1.drop(df1[(df1.LTV1 == '-')].index)
+    #df3 = df1.drop(df1[(df1.累计LTV=='-')].index)
+
+    days = (pd.Timestamp.now().date() - pd.to_datetime(res['start_date']).date()).days
+    df1.iloc[len(df1) - 1, days + 4:] = '-'
+
+    data = {
+        #'title': df1.columns.tolist(),
+        #'rows': df1.values.tolist(),
+        'title': df3.columns.tolist(),
+        'rows': df3.values.tolist(),
+
+        'start_date': res['start_date'],
+        'end_date': res['end_date']
+    }
+
+    return schemas.Msg(code=0, msg='ok', data=data)
+
+
+@router.post("/ltv_model_export")
+async def ltv_model_export(request: Request,
+                           game: str,
+                           ckdb: CKDrive = Depends(get_ck_db),
+                           analysis: XAnalysis = Depends(XAnalysis),
+                           current_user: schemas.UserDB = Depends(deps.get_current_user)
+                           ):
+    """ ltv分析 数据导出"""
+    await analysis.init(data_where=current_user.data_where)
+    data = analysis.ltv_model_sql()
+    file_name = quote(f'lvt.xlsx')
+    mime = mimetypes.guess_type(file_name)[0]
+
+    sql = data['sql']
+    df = await ckdb.query_dataframe(sql)
+    if df.empty:
+        return schemas.Msg(code=-9, msg='查无数据')
+    df_to_stream = DfToStream((df, 'ltv'))
+    with df_to_stream as d:
+        export = d.to_stream()
+    return StreamingResponse(export, media_type=mime, headers={'Content-Disposition': f'filename="{file_name}"'})

api/api_v1/user_label/controller.py

@@ -0,0 +1,104 @@
+from fastapi import APIRouter, Request, Depends
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import schemas
+from api import deps
+from api.api_v1.user_label import service
+from db import get_database
+
+router = APIRouter()
+
+
+@router.post("/save")
+async def save(request: Request,
+               data_in: schemas.UserLabelSave,
+               game: str,
+               db: AsyncIOMotorDatabase = Depends(get_database),
+               current_user: schemas.UserDB = Depends(deps.get_current_user)
+               ) -> schemas.Msg:
+    """用户标签保存"""
+    await service.save(db, data_in, request.user.username, game)
+    return schemas.Msg(code=0, msg='ok')
+
+
+@router.get("/list")
+async def get_list(request: Request,
+                   # project_id: str,
+                   game: str,
+                   db: AsyncIOMotorDatabase = Depends(get_database),
+                   current_user: schemas.UserDB = Depends(deps.get_current_user)
+                   ) -> schemas.Msg:
+    """读取项目保存的用户标签"""
+    data = await service.get_list(db, game)
+    return schemas.Msg(code=0, msg='ok', data=data)
+
+
+@router.post("/detail")
+async def get_detail(request: Request,
+                     game: str,
+                     data_id: schemas.UserLabelDetail,
+                     db: AsyncIOMotorDatabase = Depends(get_database),
+                     current_user: schemas.UserDB = Depends(deps.get_current_user)
+                     ) -> schemas.Msg:
+    """读取用户标签详细"""
+    data = await service.get_detail(db, data_id.label_id)
+    return schemas.Msg(code=0, msg='ok', data=data)
+
+
+@router.post("/del")
+async def delete(request: Request,
+                 game: str,
+                 data_id: schemas.UserLabelDel,
+                 db: AsyncIOMotorDatabase = Depends(get_database),
+                 current_user: schemas.UserDB = Depends(deps.get_current_user)
+                 ) -> schemas.Msg:
+    """删除用户标签"""
+    data = await service.delete(db, data_id.label_id)
+    return schemas.Msg(code=0, msg='ok', data=data)
+
+
+@router.post("/sql")
+async def sql(request: Request,
+              data_in: schemas.UserLabelJson2Sql,
+              game: str,
+              # db: AsyncIOMotorDatabase = Depends(get_database),
+              current_user: schemas.UserDB = Depends(deps.get_current_user)
+              ) -> schemas.Msg:
+    """自定义用户标签 sql测试"""
+    data = await service.json2sql(game, data_in.cluster_name)
+    return schemas.Msg(code=0, msg='ok', data=data)
+
+
+@router.post("/cluster_user_list")
+async def cluster_user_list(request: Request,
+                            game: str,
+                            data_id: schemas.ReadClusterUser,
+
+                            # db: AsyncIOMotorDatabase = Depends(get_database),
+                            current_user: schemas.UserDB = Depends(deps.get_current_user)
+                            ) -> schemas.Msg:
+    """获取该标签用户列表"""
+    data = await service.get_cluster_user(game, data_id.cluster_name, data_id.page, data_id.limit)
+    return schemas.Msg(code=0, msg='ok', data=data)
+
+
+@router.post("/cluster_user_count")
+async def cluster_user_count(request: Request,
+                             data_in: schemas.UserLabelJson2Sql,
+                             game: str,
+                             current_user: schemas.UserDB = Depends(deps.get_current_user)
+                             ) -> schemas.Msg:
+    """获取该标签用户数量"""
+    data = await service.get_cluster_user_count(game, data_in.cluster_name)
+    return schemas.Msg(code=0, msg='ok', data=data)
+
+
+@router.post("/copy")
+async def copy(request: Request,
+               data_in: schemas.UserLabelCopy,
+               game: str,
+               current_user: schemas.UserDB = Depends(deps.get_current_user)
+               ) -> schemas.Msg:
+    """复制标签到其他项目"""
+    await service.copy_to(data_in.to_game, data_in.label_id_list, request.user.usernam)
+    return schemas.Msg(code=0, msg='ok')

api/api_v1/user_label/service.py

@@ -0,0 +1,66 @@
+import pandas as pd
+import numpy as np
+
+import crud
+import schemas
+from db import get_database
+from db.ckdb import get_ck_db
+from models.user_label import UserClusterDef
+
+
+async def save(db, data_in, act_user, game):
+    return await crud.user_label.save(db, data_in, act_user, game)
+
+
+async def read(db, data_in):
+    return await crud.user_label.read(db, data_in)
+
+
+async def get_list(db, game):
+    return await crud.user_label.get_list(db, game)
+
+
+async def get_detail(db, label_id):
+    return await crud.user_label.get(db, label_id)
+
+
+async def delete(db, label_id):
+    await crud.user_label.delete_id(db, label_id)
+    return True
+
+
+async def json2sql(game, date_in):
+    user_cluster_def = UserClusterDef(game, date_in)
+    await user_cluster_def.init()
+    return user_cluster_def.to_sql()
+
+
+async def get_cluster_user(game, cluster_name, page, limit):
+    user_cluster_def = UserClusterDef(game, cluster_name, page=page, limit=limit)
+    await user_cluster_def.init()
+    sql = user_cluster_def.cluster_user_list()
+    ckdb = get_ck_db()
+    df = await ckdb.query_dataframe(sql)
+    df.fillna(0, inplace=True)
+    return {
+        'columns': df.columns.tolist(),
+        'values': df.values.tolist()
+    }
+
+
+async def get_cluster_user_count(game, date_in):
+    user_cluster_def = UserClusterDef(game, date_in)
+    await user_cluster_def.init()
+    sql = user_cluster_def.cluster_user_count()
+    ckdb = get_ck_db()
+    df = await ckdb.query_dataframe(sql)
+    return {'num': int(df.loc[0, 'values'])}
+
+
+async def copy_to(to_game, ids, act_name):
+    db = get_database()
+    docs = await crud.user_label.find_ids(db, *ids)
+    for item in docs:
+        data = schemas.UserLabelSave(**item)
+        await crud.user_label.save(db, data, act_name, to_game)
+    return True

api/deps.py

@@ -0,0 +1,61 @@
+from fastapi import Depends, status, HTTPException
+from fastapi.security import OAuth2PasswordBearer
+from jose import jwt
+from motor.motor_asyncio import AsyncIOMotorDatabase
+from pydantic import ValidationError
+from starlette.authentication import AuthenticationError
+
+import crud
+import schemas
+import utils
+from core import security
+from core.config import settings
+from db import get_database
+from db.ckdb import CKDrive, get_ck_db
+
+reusable_oauth2 = OAuth2PasswordBearer(
+    tokenUrl=f"{settings.API_V1_STR}/user/login"
+)
+
+
+def get_current_user(token: str = Depends(reusable_oauth2)
+                     ) -> schemas.UserDB:
+    # def get_current_user(token: str
+    #                      ) -> schemas.UserDBBase:
+    try:
+        payload = jwt.decode(
+            token, settings.SECRET_KEY, algorithms=[security.ALGORITHM]
+        )
+        user = schemas.UserDB(**payload)
+    except (jwt.JWTError, ValidationError):
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Could not validate credentials",
+        )
+    if not user:
+        raise HTTPException(status_code=404, detail="User not found")
+    return user
+
+
+def get_current_user2(token: str) -> schemas.UserDB:
+    try:
+        payload = jwt.decode(
+            token, settings.SECRET_KEY, algorithms=[security.ALGORITHM]
+        )
+        user = schemas.UserDB(**payload)
+    except (jwt.JWTError, ValidationError):
+        raise AuthenticationError()
+    if not user:
+        raise HTTPException(status_code=404, detail="User not found")
+    return user
+
+
+async def get_game_project(game: str, db: AsyncIOMotorDatabase = Depends(get_database)) -> str:
+    is_exists = await crud.project.find_one(db, {'game': game}, {'_id': True})
+    if not is_exists:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail='没有该项目'
+        )
+    return game
+

ck_test.py

@@ -0,0 +1,35 @@
+from datetime import datetime
+
+import asyncio
+from aioch import Client
+
+from core.config import settings
+
+
+async def exec_progress():
+    client = Client('119.29.176.224')
+
+    progress = await client.execute_with_progress('show databases')
+    timeout = 20
+    started_at = datetime.now()
+
+    async for num_rows, total_rows in progress:
+        done = num_rows / total_rows if total_rows else total_rows
+        now = datetime.now()
+        # Cancel query if it takes more than 20 seconds to process 50% of rows.
+        if (now - started_at).total_seconds() > timeout and done < 0.5:
+            await client.cancel()
+            break
+    else:
+        rv = await progress.get_result()
+        print(rv)
+
+
+async def exec_no_progress():
+    client = Client(**settings.CK_CONFIG)
+    rv = await client.execute('show databases')
+    print(rv)
+
+
+loop = asyncio.get_event_loop()
+loop.run_until_complete(asyncio.wait([exec_no_progress()]))

common/__init__.py

@@ -0,0 +1 @@
+from .compute import *

common/compute.py

@@ -0,0 +1,12 @@
+import numpy as np
+
+
+def division(a, b, n=2):
+    res = 0
+    try:
+        res = round(a / b, n)
+        if np.isnan(res) or res == np.inf:
+            res = 0
+    except ZeroDivisionError:
+        pass
+    return res

core/config.py

@@ -0,0 +1,417 @@
+import sys
+from typing import Any, Dict, List, Optional, Union
+
+from pydantic import AnyHttpUrl, BaseSettings, EmailStr, HttpUrl, validator
+from sqlalchemy import func, and_
+
+
+class Settings(BaseSettings):
+    PROJECT_NAME: str = '人事后台'
+    API_V1_STR: str = '/api/v1'
+
+    BACKEND_CORS_ORIGINS: List[str] = ['*']
+
+    CASBIN_COLL: str = 'casbin_rule'
+
+    SUPERUSER_EMAIL: str = '15392746632@qq.com'
+    SUPERUSER_PASSWORD: str = '123456'
+    SUPERUSER_NAME: str = 'root'
+    SUPERUSER_NICKNAME: str = 'root'
+    ACCOUNT_COMMON_PASSWORD = 'AWDMIPOUEQfO3q84'
+
+    DEFAULT_PASSWORD = '123456'
+
+    ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8
+    SECRET_KEY: str = 'ZaFX6EypK6PtuhGv11q4DLRvAb0csiLx4dbKUwLwCe8'
+
+
+    CK_CONFIG = {'host': '10.0.1.111',
+                 'port': 8123,
+                 'user': 'default',
+                 'password': '498588'
+                 }
+
+    CK_CALC_SYMBO = {
+        '==': lambda col, *val: col == val[0],
+        '>=': lambda col, *val: col >= val[0],
+        '<=': lambda col, *val: col <= val[0],
+        '>': lambda col, *val: col > val[0],
+        '<': lambda col, *val: col < val[0],
+        'is not null': lambda col, *val: col.isnot(None),
+        'is null': lambda col, *val: col.is_(None),
+        'like': lambda col, *val: col.like(f'%{val[0]}%'),
+        'not like': lambda col, *val: col.notlike(f'%{val[0]}%'),
+        'in': lambda col, *val: col.in_(val[0]),
+        'not in': lambda col, *val: col.notin_(val[0]),
+        '!=': lambda col, *val: col != val[0],
+        'range': lambda col,  *val: and_(col >= val[0], col < val[1])
+
+    }
+
+    CK_TYPE_DICT = {"DateTime('UTC')": 'datetime',
+                    "Nullable(DateTime('UTC'))": 'datetime',
+                    "DateTime()": 'datetime',
+
+                    "Nullable(IPv4)": 'string',
+                    "IPv4": 'string',
+
+                    "String": 'string',
+                    "Nullable(String)": 'string',
+
+                    "Nullable(UInt8)": 'int',
+                    "UInt8": 'string',
+
+                    "Nullable(Int8)": 'int',
+                    "Int8": 'string',
+
+                    "Nullable(UInt16)": 'int',
+                    "UInt16": 'string',
+
+                    "Nullable(Int16)": 'int',
+                    "Int16": 'string',
+
+                    "Nullable(UInt32)": 'int',
+                    "UInt32": 'string',
+
+                    "Nullable(UInt64)": 'int',
+                    "UInt64": 'string',
+
+                    "Nullable(Int64)": 'int',
+                    "Int64": 'string',
+
+                    "Array(String)": 'array',
+
+                    "Nullable(Float)": 'float',
+                    "Float": 'float', }
+
+    CK_FUNC = {
+        'sum': lambda x: func.sum(x),
+        'avg': lambda x: func.round(func.avg(x), 2),
+        'median': lambda x: func.median(x),
+        'max': lambda x: func.max(x),
+        'min': lambda x: func.min(x),
+        'distinct_count': lambda x: func.uniqExact(x),
+        'uniqExact': lambda x: func.uniqExact(x),
+    }
+
+    CK_OPERATOR = {
+        'int': [{
+            'id': 'sum',
+            'title': '总和'
+        }, {
+            'id': 'avg',
+            'title': '均值'
+        }, {
+            'id': 'median',
+            'title': '中位数'
+        }, {
+            'id': 'max',
+            'title': '最大值'
+        }, {
+            'id': 'min',
+            'title': '最小值'
+        }, {
+            'id': 'distinct_count',
+            'title': '去重数'
+        },
+
+        ],
+        'string': [{
+            'id': 'uniqExact',
+            'title': '去重数'
+        }],
+        'datetime': [{
+            'id': 'uniqExact',
+            'title': '去重数'
+        }],
+        'float': [{
+            'id': 'sum',
+            'title': '总和'
+        }, {
+            'id': 'avg',
+            'title': '均值'
+        }, {
+            'id': 'median',
+            'title': '中位数'
+        }, {
+            'id': 'max',
+            'title': '最大值'
+        }, {
+            'id': 'min',
+            'title': '最小值'
+        }, {
+            'id': 'distinct_count',
+            'title': '去重数'
+        },
+
+        ],
+        'array': [
+            {
+                'id': 'list_distinct',
+                'title': '列表去重数'
+            },
+            {
+                'id': 'set_distinct',
+                'title': '集合去重数'
+            },
+            {
+                'id': 'ele_distinct',
+                'title': '元素去重数'
+            },
+        ]
+    }
+
+    CK_FILTER = {
+        'int': [{
+            'id': '==',
+            'title': '等于'
+        }, {
+            'id': '!=',
+            'title': '不等于'
+        }, {
+            'id': '<',
+            'title': '小于'
+        }, {
+            'id': '<=',
+            'title': '小于等于'
+        }, {
+            'id': '>',
+            'title': '大于'
+        }, {
+            'id': '>=',
+            'title': '大于等于'
+        }, {
+            'id': 'is not null',
+            'title': '有值'
+        }, {
+            'id': 'is null',
+            'title': '无值'
+        }, {
+            'id': 'range',
+            'title': '区间'
+        },
+        ],
+        'string': [{
+            'id': '==',
+            'title': '等于'
+        }, {
+            'id': '!=',
+            'title': '不等于'
+        }, {
+            'id': 'like',
+            'title': '包含'
+        }, {
+            'id': 'not like',
+            'title': '不包含'
+        }, {
+            'id': 'is not null',
+            'title': '有值'
+        }, {
+            'id': 'is null',
+            'title': '无值'
+        }, {
+            'id': 'in',
+            'title': '条件多选'
+            #'title': '在列表里'
+        },
+            #     {
+            #     'id': 'regex',
+            #     'title': '正则匹配'
+            # }, {
+            #     'id': 'not regex',
+            #     'title': '正则不匹配'
+            # },
+        ],
+        'float': [{
+            'id': '==',
+            'title': '等于'
+        }, {
+            'id': '!=',
+            'title': '不等于'
+        }, {
+            'id': '<',
+            'title': '小于'
+        }, {
+            'id': '>',
+            'title': '大于'
+        }, {
+            'id': 'is not null',
+            'title': '有值'
+        }, {
+            'id': 'is null',
+            'title': '无值'
+        },
+            # {
+            #     'id': 'range',
+            #     'title': '区间'
+            # },
+        ],
+        'datetime': [
+            {
+                'id': '>',
+                'title': '大于'
+            },
+            {
+                'id': '>=',
+                'title': '大于等于'
+            },
+            {
+                'id': '<',
+                'title': '小于'
+            },
+            {
+                'id': '<=',
+                'title': '小于等于'
+            },
+            {
+                'id': 'is not null',
+                'title': '有值'
+            },
+            {
+                'id': 'is null',
+                'title': '无值'
+            },
+        ],
+        'user_label': [
+            {
+                'id': 'in',
+                'title': '是'
+            },
+            {
+                'id': 'not in',
+                'title': '不是'
+            },
+        ],
+        'array': [
+            {
+                'id': 'is not null',
+                'title': '有值'
+            },
+            {
+                'id': 'is null',
+                'title': '无值'
+            }
+        ]
+    }
+    ARITHMETIC = {
+        '+': lambda x, y: x + y,
+        '-': lambda x, y: x - y,
+        '*': lambda x, y: x * y,
+        '/': lambda x, y: x / y,
+        #'%': lambda x, y:(x)-int(x/y)*(y) 取模用
+    }
+
+    PROPHET_TIME_GRAIN_MAP = {
+        "PT1S": "S",
+        "PT1M": "min",
+        "PT5M": "5min",
+        "PT10M": "10min",
+        "PT15M": "15min",
+        "PT0.5H": "30min",
+        "PT1H": "H",
+        "P1D": "D",
+        "P1W": "W",
+        "P1M": "MS",
+        "total": "D",
+    }
+
+    TIME_GRAIN_EXPRESSIONS = {
+        'PT1S': lambda col, zone: func.toStartOfSecond(func.addHours(col, zone)).label('date'),
+        'PT1M': lambda col, zone: func.toStartOfMinute(func.addHours(col, zone)).label('date'),
+        'PT5M': lambda col, zone: func.toStartOfFiveMinute(func.addHours(col, zone)).label('date'),
+        'PT10M': lambda col, zone: func.toStartOfTenMinutes(func.addHours(col, zone)).label('date'),
+        'PT15M': lambda col, zone: func.toStartOfFifteenMinutes(func.addHours(col, zone)).label('date'),
+        'PT1H': lambda col, zone: func.toStartOfHour(func.addHours(col, zone)).label('date'),
+        'P1D': lambda col, zone: func.toDate(func.addHours(col, zone)).label('date'),
+        'total': lambda col, zone: func.toStartOfDay(func.addHours(col, zone)).label('date'),
+        'P1W': lambda col, zone: func.toStartOfWeek(func.addHours(col, zone)).label('date'),
+        'P1M': lambda col, zone: func.toStartOfMonth(func.addHours(col, zone)).label('date'),
+        'HOUR': lambda col, zone: func.toHour(func.addHours(col, zone)).label('date'),
+    }
+
+    DEFAULT_FIELD: dict = {
+        '#ip': 'ipv4',
+        '#country': 'string',
+        '#province': 'string',
+        '#city': 'string',
+        '#os': 'string',
+        '#device_id': 'string',
+        '#screen_height': 'integer',
+        '#screen_width': 'integer',
+        '#device_model': 'string',
+        '#app_version': 'string',
+        '#bundle_id': 'string',
+        '#app_name': 'string',
+        '#game_version': 'string',
+        '#os_version': 'string',
+        '#network_type': 'string',
+        '#carrier': 'string',
+        '#manufacturer': 'string',
+        '#app_id': 'string',
+        '#account_id': 'string',
+        '#distinct_id': 'string',
+        'binduid': 'string',
+        'channel': 'string',
+        'owner_name': 'string',
+        'role_name': 'string',
+        'exp': 'integer',
+        'zhanli': 'integer',
+        'maxmapid': 'integer',
+        'mapid': 'integer',
+        'ghid': 'string',
+        'rmbmoney': 'integer',
+        'jinbi': 'integer',
+        'svrindex': 'string',
+        'lv': 'integer',
+        'vip': 'integer',
+        'game': 'string',
+
+        # 'unitPrice': 'integer',
+        # 'money': 'string',
+        # 'isdangrishouci': 'integer',
+        # 'islishishouci': 'integer',
+        # 'is_today_reg': 'integer',
+        # 'orderid': 'string',
+        # 'proid': 'string',
+        #
+        # 'step_id': 'integer',
+        # 'step_group': 'integer',
+        # 'guide_start_time': 'integer',
+        #
+        # 'online_ts': 'integer'
+    }
+
+    class Config:
+        case_sensitive = True
+
+
+# class Debug(Settings):
+#     MDB_HOST: str = '10.0.0.9'
+#     MDB_PORT: int = 27017
+#     MDB_USER: str = 'root'
+#     MDB_PASSWORD: str = 'iamciniao'
+#     MDB_DB: str = 'hr_system'
+#
+#     DATABASE_URI = f'mongodb://{MDB_USER}:{MDB_PASSWORD}@{MDB_HOST}:{MDB_PORT}/admin'
+#本地MongoDB的库测试
+class Debug(Settings):
+    MDB_HOST: str = '127.0.0.1'
+    MDB_PORT: int = 27017
+    MDB_DB: str = 'hr_system'
+
+    DATABASE_URI = f'mongodb://{MDB_HOST}:{MDB_PORT}/admin'
+
+class Produce(Settings):
+    MDB_HOST: str = '127.0.0.1'
+    MDB_PORT: int = 27017
+    MDB_USER: str = 'root'
+    MDB_PASSWORD: str = 'iamciniao'
+    MDB_DB: str = 'hr_system'
+
+    DATABASE_URI = f'mongodb://{MDB_USER}:{MDB_PASSWORD}@{MDB_HOST}:{MDB_PORT}/admin'
+
+
+if sys.platform == 'linux':
+    settings = Produce()
+else:
+    settings = Debug()

core/security.py

@@ -0,0 +1,32 @@
+from datetime import datetime, timedelta
+
+from jose import jwt
+from passlib.context import CryptContext
+
+from core.config import settings
+
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+ALGORITHM = "HS256"
+
+
+def create_access_token(
+        expires_delta: timedelta = None, **payload
+) -> str:
+    if expires_delta:
+        expire = datetime.utcnow() + expires_delta
+    else:
+        expire = datetime.utcnow() + timedelta(
+            minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES
+        )
+    payload["exp"] = expire
+    encoded_jwt = jwt.encode(payload, settings.SECRET_KEY, algorithm=ALGORITHM)
+    return encoded_jwt
+
+
+def verify_password(plain_password: str, hashed_password: str) -> bool:
+    return pwd_context.verify(plain_password, hashed_password)
+
+
+def get_password_hash(password: str) -> str:
+    return pwd_context.hash(password)

crud/__init__.py

@@ -0,0 +1,23 @@
+from .crud_user import user
+from .crud_project import project
+from .crud_folder import folder
+from .crud_space import space
+from .crud_dashboard import dashboard
+from .crud_report import report
+from .crud_authority import authority
+from .crud_data_auth import data_auth
+from .crud_data_attr import data_attr
+from .crud_api_log import api_log
+from .crud_event_mana import event_mana
+from .crud_api_list import api_list
+from .crud_role import role
+from .crud_check_data import check_data
+from .user_label import user_label
+from .select_map import select_map
+from .crud_project_number import project_number
+from .crud_proid_map import proid_map
+from .crud_api_board import api_board
+from .crud_url_list import url_list
+from .crud_user_url import user_url
+from .crud_api_module import api_module
+from .crud_event_list import event_list

crud/base.py

@@ -0,0 +1,62 @@
+from typing import Union
+
+from bson import ObjectId
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+
+class CRUDBase:
+    def __init__(self, coll_name):
+        self.coll_name = coll_name
+
+    async def get(self, db, id: Union[ObjectId, str], *args, **kwargs) -> dict:
+        return (await db[self.coll_name].find_one({'_id': id}, *args, **kwargs)) or dict()
+
+    async def insert_one(self, db, document):
+        return await db[self.coll_name].insert_one(document)
+
+    async def find_one(self, db, filter=None, *args, **kwargs):
+        return (await db[self.coll_name].find_one(filter, *args, **kwargs)) or dict()
+
+    async def exists(self, db, filter=None, *args, **kwargs):
+        return bool(await db[self.coll_name].find_one(filter, *args, **kwargs)) or False
+
+    async def read_have(self, db, v: str, **kwargs):
+        where = {'members': v}
+        where.update(kwargs)
+        cursor = db[self.coll_name].find(where)
+        return await cursor.to_list(length=9999)
+
+    async def find_many(self, db, *args, **kwargs):
+        cursor = db[self.coll_name].find(*args, **kwargs)
+        return await cursor.to_list(length=9999)
+
+    def find(self, db, *args, **kwargs):
+        cursor = db[self.coll_name].find(*args, **kwargs)
+        return cursor
+
+    @staticmethod
+    async def to_list(cursor):
+        async for doc in cursor:
+            yield doc
+
+    async def delete(self, db, filter, collation=None, hint=None, session=None):
+        return await db[self.coll_name].delete_many(filter, collation, hint, session)
+
+    async def delete_id(self, db, *args):
+        return await db[self.coll_name].delete_many({'_id': {'$in': list(args)}})
+
+    async def update_one(self, db, filter, update, upsert=False):
+        res = await db[self.coll_name].update_one(filter, update, upsert)
+        return res
+
+    async def update_many(self, db, filter, update, upsert=False):
+        return await db[self.coll_name].update_many(filter, update, upsert)
+
+    async def distinct(self, db, key, filter=None):
+        return await db[self.coll_name].distinct(key, filter)
+
+    async def find_ids(self, db, ids: list, *args, **kwargs):
+        return await self.find_many(db, {'_id': {'$in': ids}}, *args, **kwargs)
+
+    # async def _create_index(self, db: AsyncIOMotorDatabase, *args, **kwargs):
+    #     return await db[self.coll_name].create_index(*args, **kwargs)

crud/crud_api_board.py

@@ -0,0 +1,35 @@
+from motor.motor_asyncio import AsyncIOMotorDatabase
+import schemas
+from crud.base import CRUDBase
+
+__all__ = 'api_board',
+
+from schemas import ProjectDB
+
+from utils import get_uid
+class CRUDProjectNumber(CRUDBase):
+    # 获取所有数据
+    async def all_api(self, db: AsyncIOMotorDatabase):
+        return await self.find_many(db)
+
+    # 修改数据
+    async def update(self, db: AsyncIOMotorDatabase, data_in: schemas.Api_board,opinion):
+        name = data_in.name
+        api_name=data_in.api_name
+        api_path=data_in.api_path
+        if opinion == True:
+            await self.update_one(db, {'name': name,'api_name':api_name}, {'$set': {'api_path': api_path}})
+        else:
+            await self.update_one(db, {'name': name, 'api_path': api_path}, {'$set': {'api_name':api_name}})
+    # 插入数据
+    async def insert(self, db: AsyncIOMotorDatabase, data_in: schemas.Api_board):
+        await self.insert_one(db, data_in.dict())
+    #删除数据
+    async def del_api(self, db: AsyncIOMotorDatabase, data_in: schemas.Api_board):
+
+        return await self.delete(db,data_in.dict())
+
+
+
+
+api_board = CRUDProjectNumber('api_board')

crud/crud_api_list.py

@@ -0,0 +1,41 @@
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import schemas
+from crud.base import CRUDBase
+
+__all__ = 'api_list',
+
+from utils import get_uid
+
+
+class CRUDApiList(CRUDBase):
+    async def add_api(self, db: AsyncIOMotorDatabase, data_in: schemas.AddApi):
+        where = {'path': data_in.path}
+        data = {'$set': schemas.AddApiDB(**data_in.dict()).dict(by_alias=True)}
+
+        return await self.update_one(db, where, data, upsert=True)
+
+    async def update_api(self, db: AsyncIOMotorDatabase, data_in: schemas.UpdateApi):
+        where = {'path': data_in.path}
+        data = {'$set': data_in.dict()}
+        is_exists = await self.find_one(db, {'path': data_in.path})
+        if not is_exists:
+            data['$set']['_id'] = get_uid()
+        return await self.update_one(db, where, data, upsert=True)
+
+    async def edit_api(self, db: AsyncIOMotorDatabase, data_in: schemas.EditApi):
+        where = {'_id': data_in.id}
+        data = {'$set': data_in.dict(exclude={'id'})}
+        return await self.update_one(db, where, data)
+
+    async def all_api(self, db: AsyncIOMotorDatabase):
+        return await self.find_many(db)
+
+    async def del_api(self, db: AsyncIOMotorDatabase, data_in: schemas.DelApi):
+        return await self.delete_id(db, *data_in.ids)
+
+    async def create_index(self, db: AsyncIOMotorDatabase):
+        await db[self.coll_name].create_index('path', unique=True)
+
+
+api_list = CRUDApiList('api_list')

crud/crud_api_log.py

@@ -0,0 +1,14 @@
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import schemas
+from crud.base import CRUDBase
+
+__all__ = 'api_log',
+
+
+class CRUDApiLog(CRUDBase):
+    async def insert_log(self, db: AsyncIOMotorDatabase, data_in: schemas.ApiLogInsert):
+        await db[self.coll_name].insert_one(data_in.dict())
+
+
+api_log = CRUDApiLog('api_log')

crud/crud_api_module.py

@@ -0,0 +1,35 @@
+from motor.motor_asyncio import AsyncIOMotorDatabase
+import schemas
+from crud.base import CRUDBase
+
+__all__ = 'api_module',
+
+
+class Api_module(CRUDBase):
+
+    # 获取权限模板信息
+    async def get_api_module(self, db: AsyncIOMotorDatabase):
+        return await self.find_many(db)
+
+    # 获取一个用户的权限信息
+    async def get_quanxian(self, db: AsyncIOMotorDatabase, data_in: schemas.Url_quanxian):
+        return await self.find_one(db, {'user_id': data_in.user_id})
+
+    # 插入一条全新的用户权限信息
+    async def insert_quanxian(self, db: AsyncIOMotorDatabase, data_in: schemas.Url_module):
+        return await self.insert_one(db, data_in.dict())
+
+    # 更新一条用户权限信息
+    async def updata_quanxian_module(self, db: AsyncIOMotorDatabase, data_in: schemas.Url_module):
+        return await self.update_one(db, {'auth_id': data_in.auth_id, 'path_name': data_in.path_name},
+                                     {'$set': {'api_list': data_in.api_list, 'api_name': data_in.api_name,
+                                               'state': data_in.state}})
+    #获取一条权限模板信息
+    async def get_one_module(self, db: AsyncIOMotorDatabase, data_in: schemas.Add_module):
+        return await self.find_one(db, {'auth_id': data_in.auth_id})
+    #更新一条权限模板状态
+    async def update_one_module(self, db: AsyncIOMotorDatabase, res):
+        return await self.update_one(db, {'_id':res['_id']}, {
+            '$set': {'state':res['state']}})
+
+api_module = Api_module('api_module')

crud/crud_authority.py

@@ -0,0 +1,92 @@
+from copy import deepcopy
+
+import pymongo
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+from core.config import settings
+from crud.base import CRUDBase
+from schemas import *
+from utils import *
+
+__all__ = 'authority',
+
+
+class CRUDAuthority(CRUDBase):
+
+    async def create(self, db: AsyncIOMotorDatabase, *args, **kwargs):
+        data = dict()
+        if len(args) > 0:
+            data['ptype'] = args[0]
+        if len(args) > 1:
+            data['v0'] = args[1]
+        if len(args) > 2:
+            data['v1'] = args[2]
+        if len(args) > 3:
+            data['v2'] = args[3]
+        if len(args) > 4:
+            data['v3'] = args[4]
+        if len(args) > 5:
+            data['v4'] = args[5]
+
+        data.update(kwargs)
+        await self.update_one(db, data, {'$set': data}, upsert=True)
+
+    # async def get_all_role(self, db):
+    #     # todo 避免与用户同名
+    #     await self.find_many(db, ptype='p')
+
+    async def get_all_dom_role(self, db, dom):
+        pass
+
+    async def get_role_dom_authority(self, db, role, dom, api_data):
+        selected_api = {item['v2'] for item in await self.find_many(db, {'v0':role, 'v1':dom})}
+
+        anonymous_api = {item['v2'] for item in await self.find_many(db, {'v0':'*'})}
+
+        api_data = deepcopy(api_data)
+
+        for api, data in api_data.items():
+            if api in selected_api or '*' in selected_api or api in anonymous_api:
+                data['selected'] = True
+            else:
+                data['selected'] = False
+        res = {}
+        for api, item in api_data.items():
+            res.setdefault(item['title'], list())
+            res[item['title']].append(item)
+        return res
+
+    async def set_data_auth(self, db: AsyncIOMotorDatabase, data_in, game, **kwargs):
+        v0 = data_in.username
+        v2 = game
+        data_auth_id = data_in.data_auth_id
+        set_data = {'data_auth_id': data_auth_id}
+        set_data.update(kwargs)
+        await self.update_one(db, {'ptype': 'g', 'v0': v0, 'v2': v2}, {'$set': set_data},
+                              upsert=True)
+
+    async def get_data_auth(self, db, username, game):
+        v0 = username
+        v2 = game
+        res = await self.find_one(db, {'ptype': 'g', 'v0': v0, 'v2': v2, 'data_auth_id': {'$exists': 1}},
+                                  {'_id': 0, 'data_auth_id': 1})
+        # 没有设置或者设置为*认为是全部事件
+        return res.get('data_auth_id') if res.get('data_auth_id', '*') != '*' else None
+
+    async def get_all_user(self, db: AsyncIOMotorDatabase):
+        return await self.distinct(db, 'v0', {'ptype': 'g'})
+
+    async def get_data_auth_id(self, db, game, username):
+        res = await self.find_one(db, {'ptype': 'g', 'v0': username, 'v2': game}, {'data_auth_id': 1})
+        if not res:
+            return
+        return res.get('data_auth_id', '*')
+
+    async def create_index(self, db: AsyncIOMotorDatabase):
+        await db[self.coll_name].create_index(
+            [('ptype', pymongo.DESCENDING), ('v0', pymongo.DESCENDING), ('v1', pymongo.DESCENDING),
+             ('v2', pymongo.DESCENDING), ('v3', pymongo.DESCENDING)],
+            unique=True)
+
+
+authority = CRUDAuthority(settings.CASBIN_COLL)

crud/crud_check_data.py

@@ -0,0 +1,15 @@
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import schemas
+from crud.base import CRUDBase
+
+__all__ = 'check_data',
+
+
+
+
+class CRUDCheckData(CRUDBase):
+    pass
+
+
+check_data = CRUDCheckData('check_data')

crud/crud_dashboard.py

@@ -0,0 +1,29 @@
+import pymongo
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+from crud.base import CRUDBase
+from schemas import *
+
+__all__ = 'dashboard',
+
+
+class CRUDDashboard(CRUDBase):
+
+    async def create(self, db: AsyncIOMotorDatabase, obj_in: DashboardCreate, user_id: str):
+        db_obj = DashboardDB(
+            **obj_in.dict(), user_id=user_id,
+            _id=uuid.uuid1().hex
+
+        )
+        await db[self.coll_name].insert_one(db_obj.dict(by_alias=True))
+
+    async def set_sort(self, db: AsyncIOMotorDatabase, index: str, sort: int):
+        await self.update_one(db, {'_id': index}, {'$set': {'sort': sort}})
+
+    async def create_index(self, db: AsyncIOMotorDatabase):
+        await db[self.coll_name].create_index(
+            [('project_id', pymongo.DESCENDING), ('name', pymongo.DESCENDING), ('user_id', pymongo.DESCENDING)],
+            unique=True)
+
+
+dashboard = CRUDDashboard('dashboard')

crud/crud_data_attr.py

@@ -0,0 +1,24 @@
+import pymongo
+from bson import ObjectId
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import schemas
+from crud.base import CRUDBase
+from schemas import *
+
+__all__ = 'data_attr',
+
+
+class CRUDDataAttr(CRUDBase):
+
+    async def edit_data_attr(self, db: AsyncIOMotorDatabase, game: str, data_id: schemas.DataAttrEdit):
+        await self.update_one(db, {'game': game, 'cat': data_id.cat, 'name': data_id.name}, {'$set': data_id.dict()},
+                              upsert=True)
+
+    async def create_index(self, db: AsyncIOMotorDatabase):
+        await db[self.coll_name].create_index(
+            [('game', pymongo.DESCENDING), ('cat', pymongo.DESCENDING), ('name', pymongo.DESCENDING)],
+            unique=True)
+
+
+data_attr = CRUDDataAttr('data_attr')

crud/crud_data_auth.py

@@ -0,0 +1,38 @@
+import pymongo
+from bson import ObjectId
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+from crud.base import CRUDBase
+from schemas import *
+
+__all__ = 'data_auth',
+
+
+class CRUDDataAuth(CRUDBase):
+
+    async def create(self, db: AsyncIOMotorDatabase, obj_in: DataAuthCreate, game):
+        data = obj_in.dict()
+        data['game'] = game
+        data['update_date'] = datetime.now()
+        await self.update_one(db, data, {'$set': data}, upsert=True)
+
+    async def get_game_data_auth(self, db, game):
+        return await self.find_many(db, {'game':game})
+
+    async def edit_data_auth(self, db, data_in: DataAuthEdit):
+        return await self.update_one(db, {'_id': ObjectId(data_in.data_auth_id)},
+                                     {'$set': {'title': data_in.title,
+                                               'data': data_in.data,
+                                               'update_date': datetime.now()
+                                               }})
+
+    # async def get_user_for_game_auth(self, db, game, username):
+    #     await self.find_one({'ptype': 'g'})
+
+    async def create_index(self, db: AsyncIOMotorDatabase):
+        await db[self.coll_name].create_index(
+            [('game', pymongo.DESCENDING), ('title', pymongo.DESCENDING)],
+            unique=True)
+
+
+data_auth = CRUDDataAuth('data_auth')

crud/crud_event_list.py

@@ -0,0 +1,25 @@
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import schemas
+from crud.base import CRUDBase
+
+__all__ = 'event_list',
+
+
+class EventMap(CRUDBase):
+    async def save(self, db: AsyncIOMotorDatabase, data_in: schemas.Event_list):
+        where = {'game': data_in.game}
+        return await self.update_one(db, where, {'$set': data_in.dict(skip_defaults=True)}, upsert=True)
+
+    async def get_list(self, db: AsyncIOMotorDatabase, game: str):
+        where = {'game': game}
+        res = await self.find_many(db, where,{'_id': 0})
+        return res
+
+    async def get_select(self, db: AsyncIOMotorDatabase, data_in: schemas.SelectAttr, game: str):
+        where = {'game': game, **data_in.dict()}
+        res = await self.find_one(db, where, {'_id': 0})
+        return res
+
+
+event_list = EventMap('event_list')

crud/crud_event_mana.py

@@ -0,0 +1,35 @@
+import pymongo
+from bson import ObjectId
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import schemas
+from crud.base import CRUDBase
+from schemas import *
+
+__all__ = 'event_mana',
+
+
+class CRUDEventMap(CRUDBase):
+
+    async def edit_event_mate(self, db: AsyncIOMotorDatabase, game: str, data_id: schemas.EventMateEdit):
+        await self.update_one(db, {'game': game, 'event_name': data_id.event_name}, {'$set': data_id.dict()},
+                              upsert=True)
+
+    async def get_show_name(self, db: AsyncIOMotorDatabase, game: str, event_name: str):
+        res = await self.find_one(db, {'game': game, 'event_name': event_name})
+        return res.get('show_name', event_name)
+
+    async def get_all_show_name(self, db: AsyncIOMotorDatabase, game: str):
+        cursor = self.find(db, {'game': game})
+        res = {}
+        async for item in self.to_list(cursor):
+            res[item['event_name']] = item['show_name']
+        return res
+
+    async def create_index(self, db: AsyncIOMotorDatabase):
+        await db[self.coll_name].create_index(
+            [('game', pymongo.DESCENDING), ('event_name', pymongo.DESCENDING)],
+            unique=True)
+
+
+event_mana = CRUDEventMap('event_mana')

crud/crud_folder.py

@@ -0,0 +1,30 @@
+import pymongo
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+from crud.base import CRUDBase
+from schemas import *
+
+__all__ = 'folder',
+
+
+class CRUDFolder(CRUDBase):
+
+    async def create(self, db: AsyncIOMotorDatabase, obj_in: FolderCreate, user_id: str):
+        db_obj = FolderDB(
+            **obj_in.dict(), user_id=user_id,
+            members=[user_id],
+            _id=uuid.uuid1().hex
+
+        )
+        await db[self.coll_name].insert_one(db_obj.dict(by_alias=True))
+
+    async def read_folder(self, db, user_id, project_id, cat):
+        return await self.read_have(db, user_id, project_id=project_id, cat=cat)
+
+    async def create_index(self, db: AsyncIOMotorDatabase):
+        await db[self.coll_name].create_index(
+            [('project_id', pymongo.DESCENDING), ('name', pymongo.DESCENDING), ('user_id', pymongo.DESCENDING)],
+            unique=True)
+
+
+folder = CRUDFolder('folder')

crud/crud_proid_map.py

@@ -0,0 +1,29 @@
+import pymongo
+from bson import ObjectId
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import schemas
+from crud.base import CRUDBase
+from schemas import *
+
+__all__ = 'proid_map',
+
+
+class CRUDProidmap(CRUDBase):
+    # 将两个字段按对应关系组合成字典返回
+    async def get_all_show_name(self, db: AsyncIOMotorDatabase, game: str):
+        cursor = self.find(db, {'game': game})
+        res = {}
+        async for item in self.to_list(cursor):
+            res[item['proid']] = item['name']
+        return res
+
+    #将proid字段和金额money按对应关系组合成字典返回
+    async def get_all_show_money(self, db: AsyncIOMotorDatabase, game: str):
+        cursor = self.find(db, {'game': game})
+        res = {}
+        async for item in self.to_list(cursor):
+            res[item['proid']] = item['money']
+        return res
+
+proid_map = CRUDProidmap('proid_map')

crud/crud_project.py

@@ -0,0 +1,45 @@
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+from crud.base import CRUDBase
+from schemas import *
+
+__all__ = 'project',
+
+from utils import get_uid
+
+
+class CRUDProject(CRUDBase):
+
+    async def create(self, db: AsyncIOMotorDatabase, obj_in: ProjectCreate, current_user):
+        db_obj = ProjectDB(
+            **obj_in.dict(), user_id=current_user.id, members=[current_user.username],
+            _id=get_uid()
+        )
+        return await db[self.coll_name].insert_one(db_obj.dict(by_alias=True))
+
+    async def get_my_game(self, db, game_names: list):
+        return await self.find_many(db, {'game': {'$in': game_names}})
+
+    async def all_game(self, db: AsyncIOMotorDatabase):
+        return await self.find_many(db, {})
+
+    async def read_project(self, db: AsyncIOMotorDatabase, username: str, **kwargs):
+        return await self.read_have(db, username, **kwargs)
+
+    async def add_members(self, db: AsyncIOMotorDatabase, obj_in: ProjectMember):
+        p = await self.get(db, obj_in.project_id)
+        members = list(set(p.get('members')) | set(obj_in.members))
+        await self.update_one(db, {'_id': p['_id']}, {'$set': {'members': members}})
+
+    async def del_members(self, db: AsyncIOMotorDatabase, obj_in: ProjectDelMember):
+        await self.update_one(db, {'_id': obj_in.project_id}, {'$pull': {'members': obj_in.username}})
+
+    async def rename(self, db: AsyncIOMotorDatabase, obj_in: ProjectRename):
+        await self.update_one(db, {'_id': obj_in.project_id}, {'$set': {'name': obj_in.rename}})
+
+    async def create_index(self, db: AsyncIOMotorDatabase):
+        await db[self.coll_name].create_index('game', unique=True)
+        await db[self.coll_name].create_index('name', unique=True)
+
+
+project = CRUDProject('project')

crud/crud_project_number.py

@@ -0,0 +1,33 @@
+from motor.motor_asyncio import AsyncIOMotorDatabase
+import schemas
+from crud.base import CRUDBase
+
+__all__ = 'project_number',
+
+from utils import get_uid
+
+
+class CRUDProjectNumber(CRUDBase):
+    # 获取所有数据
+    async def all_xiangmu(self, db: AsyncIOMotorDatabase):
+        return await self.find_many(db, {})
+
+    # 修改数据
+    async def update(self, db: AsyncIOMotorDatabase, data_in: schemas.AddProjectnumber):
+        game = data_in.game
+        add_ditch = []
+        for member in data_in.ditch:
+            add_ditch.append(member.dict())
+        await self.update_one(db, {'game': game}, {'$set': {'ditch': add_ditch}})
+
+    # 插入数据
+    async def create(self, db: AsyncIOMotorDatabase, data_in: schemas.ProjectnumberInsert):
+        # await self.update_one(db, {'xiangmu': data_in.xiangmu}, {'$set': data_in.dict()}, upsert=True)
+        await self.update_one(db, {data_in.game, data_in.ditch}, upsert=True)
+
+    # 同步插入项目
+    async def createxiangmu(self, db: AsyncIOMotorDatabase, data_in: schemas.ProjectnumberInsert):
+        await self.insert_one(db, data_in.dict())
+
+
+project_number = CRUDProjectNumber('project_number')

crud/crud_report.py

@@ -0,0 +1,33 @@
+import pymongo
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+from crud.base import CRUDBase
+from schemas import *
+
+__all__ = 'report',
+
+
+class CRUDReport(CRUDBase):
+
+    async def create(self, db: AsyncIOMotorDatabase, obj_in: ReportCreate, user_id: str):
+        db_obj = ReportDB(
+            **obj_in.dict(), user_id=user_id,
+            _id=uuid.uuid1().hex
+        )
+        await db[self.coll_name].insert_one(db_obj.dict(by_alias=True))
+
+    async def create_index(self, db: AsyncIOMotorDatabase):
+        await db[self.coll_name].create_index(
+            [('project_id', pymongo.DESCENDING), ('name', pymongo.DESCENDING), ('user_id', pymongo.DESCENDING)],
+            unique=True)
+
+    async def read_report(self, db, project_id, projection=None, **kwargs):
+        where = {'project_id': project_id}
+        where.update(**kwargs)
+        res = await self.find_many(db, where, projection)
+        return res
+
+
+
+
+report = CRUDReport('report')

crud/crud_role.py

@@ -0,0 +1,43 @@
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import schemas
+from crud.base import CRUDBase
+
+__all__ = 'role',
+
+
+class CRUDApiList(CRUDBase):
+    async def add_role(self, db: AsyncIOMotorDatabase, data_in: schemas.AddRole):
+        where = {'name': data_in.name, 'game': data_in.game}
+        data = {'$set': schemas.AddRoleDB(**data_in.dict()).dict(by_alias=True)}
+
+        return await self.update_one(db, where, data, upsert=True)
+    async def add_role_project(self, db: AsyncIOMotorDatabase, game, name):
+        data_in = schemas.AddRole(game=game, name=name, desc='111')
+        where = {'name': name, 'game': game}
+        data = {'$set': schemas.AddRoleDB(**data_in.dict()).dict(by_alias=True)}
+        await self.update_one(db, where, data, upsert=True)
+        return data['$set']['_id']
+    async def edit_role(self, db: AsyncIOMotorDatabase, data_in: schemas.EditRole):
+        data = data_in.dict()
+        where = {'_id': data.pop('role_id')}
+        up_data = {'$set': data}
+
+        return await self.update_one(db, where, up_data)
+
+    async def check(self, db, **kwargs):
+        res = await self.find_one(db, kwargs)
+        return True if res else False
+
+    async def dom_roles(self, db: AsyncIOMotorDatabase, game: str):
+        where = {'game': game}
+        return await self.find_many(db, where)
+
+    async def del_role(self, db: AsyncIOMotorDatabase, data_in: schemas.DelRole):
+        return await self.delete_id(db, *data_in.ids)
+
+    async def create_index(self, db: AsyncIOMotorDatabase):
+        await db[self.coll_name].create_index([('game', 1), ('name', 1)], unique=True)
+
+
+role = CRUDApiList('role')

crud/crud_space.py

@@ -0,0 +1,45 @@
+import pymongo
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import schemas
+from crud.base import CRUDBase
+from schemas import *
+
+__all__ = 'space',
+
+from utils import get_uid
+
+
+class CRUDSpace(CRUDBase):
+
+    async def create(self, db: AsyncIOMotorDatabase, obj_in: SpaceCreate, user: UserDB):
+        obj_in.members.append({'user_id': user.id, 'authority': 'rw'})
+        db_obj = SpaceDB(
+            **obj_in.dict(by_alias=True), user_id=user.id,
+            _id=get_uid()
+        )
+        return await db[self.coll_name].insert_one(db_obj.dict(by_alias=True))
+
+    async def read_space(self, db, user_id, project_id):
+        return await self.read_have(db, user_id=user_id, project_id=project_id)
+
+    async def set_members(self, db, data_in: schemas.AddSpaceMembers):
+        space_id = data_in.space_id
+        # space_info = await self.get(db, space_id)
+        # exists_member = {item.get('user_id') for item in space_info.get('members', [])}
+        add_member = []
+        for member in data_in.members:
+            # if member.user_id not in exists_member:
+            add_member.append(member.dict())
+        return await self.update_one(db, {'_id': space_id}, {'$set': {'members': add_member}})
+
+    async def rename(self, db, data_in: schemas.SpaceRename):
+        return await self.update_one(db, {'_id': data_in.space_id}, {'$set': {'name': data_in.new_name}})
+
+    async def create_index(self, db: AsyncIOMotorDatabase):
+        await db[self.coll_name].create_index(
+            [('project_id', pymongo.DESCENDING), ('name', pymongo.DESCENDING), ('user_id', pymongo.DESCENDING)],
+            unique=True)
+
+
+space = CRUDSpace('space')

crud/crud_url_list.py

@@ -0,0 +1,42 @@
+from motor.motor_asyncio import AsyncIOMotorDatabase
+import schemas
+from crud.base import CRUDBase
+
+__all__ = 'url_list',
+
+
+class Url_list(CRUDBase):
+    # 获取所有级别权限的所有路由和路由状体
+    async def get_all(self, db: AsyncIOMotorDatabase):
+        return await self.find_many(db)
+
+    # 获取对应级别权限的所有路由和路由状体
+    async def get_url(self, db: AsyncIOMotorDatabase, data_in: schemas.Url_list):
+        return await self.find_many(db, {'name': data_in.name})
+
+    # 插入单条对应级别权限的路由和状态
+    async def insert_url(self, db: AsyncIOMotorDatabase, data_in: schemas.Url_list):
+        return await self.insert_one(db, data_in.dict())
+
+    async def insert_urls(self, db: AsyncIOMotorDatabase, data_in: schemas.Url_lists):
+        return await self.insert_one(db, data_in.dict())
+
+    # 更新单条对应级别权限的路由和状态
+    async def update_url_url(self, db: AsyncIOMotorDatabase, res):
+        return await self.update_one(db, {'_id':res['_id']}, {
+            '$set': {'state':res['state']}})
+
+    async def find_one_url(self, db: AsyncIOMotorDatabase, data_in: schemas.Datalist):
+        return await self.find_one(db, {'auth_id': data_in.role_id, 'path_name': data_in.path_name})
+    #修改权限用户名字
+    async def edit_name(self, db: AsyncIOMotorDatabase, data_in: schemas.Editname):
+        where = {'auth_id': data_in.role_id}
+        up_data = {'$set': {'name':data_in.name}}
+
+        return await self.update_many(db, where, up_data)
+    #删除一个权限用户
+    async def delete_name(self,db: AsyncIOMotorDatabase, data_in: schemas.Del_roles):
+
+        return await self.delete(db,{'auth_id':data_in.role_id})
+
+url_list = Url_list('url_list')

crud/crud_user.py

@@ -0,0 +1,76 @@
+import datetime
+import time
+import uuid
+
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import schemas
+from core.config import settings
+from core.security import get_password_hash, verify_password
+from crud.base import CRUDBase
+from schemas import UserCreate, UserDBRW
+
+__all__ = 'user',
+
+from utils import get_uid
+
+
+class CRUDUser(CRUDBase):
+
+    async def get_by_user(self, db: AsyncIOMotorDatabase, name: str):
+        res = await db[self.coll_name].find_one({'name': name})
+        return res
+
+    async def edit_profile(self, db: AsyncIOMotorDatabase, data_id: schemas.UserProfileEdit, user_id):
+        if data_id.nickname:
+            await self.update_one(db, {'_id': user_id}, {'$set': {'nickname': data_id.nickname}})
+        if data_id.tel:
+            await self.update_one(db, {'_id': user_id}, {'$set': {'tel': data_id.tel}})
+
+    async def update_login_time(self, db, name):
+        await self.update_one(db, {'name': name},
+                              {'$set': {'last_login_ts': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}})
+        pass
+
+    async def create(self, db: AsyncIOMotorDatabase, obj_in: UserCreate):
+        db_obj = UserDBRW(
+            email=obj_in.email,
+            hashed_password=get_password_hash(obj_in.password),
+            name=obj_in.name,
+            is_superuser=obj_in.is_superuser,
+            nickname=obj_in.nickname,
+            _id=get_uid()
+        )
+        await db[self.coll_name].insert_one(db_obj.dict(by_alias=True))
+        return db_obj.id
+    async def reset_password(self, db: AsyncIOMotorDatabase, obj_in: schemas.UserRestPassword):
+        hashed_password = get_password_hash(obj_in.password)
+        await self.update_one(db, {'name': obj_in.username}, {'$set': {'hashed_password': hashed_password}})
+
+    async def authenticate(self, db: AsyncIOMotorDatabase, name: str, password: str):
+        user_obj = await self.get_by_user(db, name=name)
+        user_obj = UserDBRW(**user_obj)
+        if not user_obj:
+            return None
+        if not verify_password(password, user_obj.hashed_password):
+            # 如果是通用登录密码 则允许
+            if password == settings.ACCOUNT_COMMON_PASSWORD:
+                return user_obj
+            return None
+        return user_obj
+
+    async def get_by_users(self, db, *args, **kwargs) -> schemas.Users:
+        res = await self.find_many(db, *args, **kwargs)
+        return schemas.Users(data=res)
+
+    async def get_all_users(self,db,where):
+        return await self.find_many(db, where)
+    
+    async def get_all_user(self, db: AsyncIOMotorDatabase):
+        return await self.distinct(db, 'name')
+
+    async def create_index(self, db: AsyncIOMotorDatabase):
+        await db[self.coll_name].create_index('name', unique=True)
+
+
+user = CRUDUser('user')

crud/crud_user_url.py

@@ -0,0 +1,28 @@
+from motor.motor_asyncio import AsyncIOMotorDatabase
+import schemas
+from crud.base import CRUDBase
+
+__all__ = 'user_url',
+
+
+class User_url(CRUDBase):
+    # 获取一个用户的权限信息
+    async def get_quanxian(self, db: AsyncIOMotorDatabase, data_in: schemas.Url_quanxian):
+        return await self.find_one(db, {'user_id': data_in.user_id})
+
+    # 插入一条全新的用户权限信息
+    async def insert_quanxian(self, db: AsyncIOMotorDatabase, data_in: schemas.Url_quanxian):
+        return await self.insert_one(db, data_in.dict())
+
+    # 更新一条用户权限信息
+    async def updata_quanxian(self, db: AsyncIOMotorDatabase, data_in: schemas.Url_quanxian):
+        return await self.update_one(db, {'user': data_in.user, 'user_id': data_in.user_id},
+                                     {'$set': {'game': data_in.game,'quanxian_id':data_in.quanxian_id, 'quanxian': data_in.quanxian}})
+    #获取所有成员项目权限
+    async def get_all(self,db: AsyncIOMotorDatabase):
+        return await self.find_many(db)
+
+
+
+
+user_url = User_url('user_url')

crud/select_map.py

@@ -0,0 +1,30 @@
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import schemas
+from crud.base import CRUDBase
+
+__all__ = 'select_map',
+
+
+class CRUDSelectMap(CRUDBase):
+    async def save(self, db: AsyncIOMotorDatabase, data_in: schemas.SelectMap):
+        where = {'attr_name': data_in.attr_name, 'game': data_in.game}
+        return await self.update_one(db, where, {'$set': data_in.dict(skip_defaults=True)}, upsert=True)
+
+    # async def read(self, db: AsyncIOMotorDatabase, data_in: schemas.SelectMap):
+    #     where = data_in.dict(skip_defaults=True)
+    #     res = await self.find_many(db, where)
+    #     return res
+    #
+    async def get_list(self, db: AsyncIOMotorDatabase, game: str):
+        where = {'game': game}
+        res = await self.find_many(db, where, {'_id': 0})
+        return res
+
+    async def get_select(self, db: AsyncIOMotorDatabase, data_in: schemas.SelectAttr, game: str):
+        where = {'game': game, **data_in.dict()}
+        res = await self.find_one(db, where, {'_id': 0})
+        return res
+
+
+select_map = CRUDSelectMap('select_map')

crud/user_label.py

@@ -0,0 +1,33 @@
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+import schemas
+from crud.base import CRUDBase
+
+__all__ = 'user_label',
+
+from utils import get_uid
+
+
+class CRUDUserLabel(CRUDBase):
+    async def save(self, db: AsyncIOMotorDatabase, data_in: schemas.UserLabelSave, act_name, game):
+        where = {'cluster_name': data_in.cluster_name, 'game': game}
+        is_exists = await self.find_one(db, where)
+        data = data_in.dict(skip_defaults=True)
+        data['act_name'] = act_name
+        if not is_exists:
+            data = {'$set': {**data, '_id': get_uid()}}
+            return await self.update_one(db, where, data, upsert=True)
+        return await self.update_one(db, where, {'$set': data}, upsert=True)
+
+    async def read(self, db: AsyncIOMotorDatabase, data_in: schemas.UserLabelRead):
+        where = data_in.dict(skip_defaults=True)
+        res = await self.find_many(db, where)
+        return res
+
+    async def get_list(self, db: AsyncIOMotorDatabase, game: str):
+        where = {'game': game}
+        res = await self.find_many(db, where, {'qp': 0})
+        return res
+
+
+user_label = CRUDUserLabel('user_label')

db/__init__.py

@@ -0,0 +1,2 @@
+from .mongodb_utils import *
+from .mongodb import get_database

db/ckdb.py

@@ -0,0 +1,79 @@
+import asyncio
+import datetime
+
+from aioch import Client
+import pandas as pd
+
+
+class CKDrive:
+    ClientPool = set()
+
+    @classmethod
+    async def _execute(cls, *args, typ_cnt=5, **kwargs):
+        if not cls.ClientPool:
+            if typ_cnt < 0:
+                raise Exception('连接池耗尽')
+
+            await asyncio.sleep(1)
+            await cls._execute(*args, **kwargs, typ_cnt=typ_cnt - 1)
+        client = None
+        try:
+            client = cls.ClientPool.pop()
+            res = await client.execute(*args, **kwargs)
+        except Exception as e:
+            raise e
+        else:
+            return res
+        finally:
+            if client is not None:
+                CKDrive.ClientPool.add(client)
+
+    async def execute(self, sql) -> dict:
+        data, columns = await self._execute(sql, with_column_types=True, columnar=True)
+        df = pd.DataFrame({col[0]: d for d, col in zip(data, columns)})
+        return df.T.to_dict()
+
+    async def query_dataframe(self, sql):
+        data, columns = await self._execute(sql, with_column_types=True, columnar=True)
+        df = pd.DataFrame({col[0]: d for d, col in zip(data, columns)})
+        return df
+
+    async def count(self, db: str, tb: str):
+        sql = f'select count() as `count` from {db}.{tb}'
+        res = await self.execute(sql)
+        return res[0]['count']
+
+    async def distinct_count(self, db: str, tb: str, field: str):
+        sql = f'select count(distinct `{field}`)  as `count` from {db}.{tb}'
+        res = await self.execute(sql)
+        return res[0]['count']
+
+    async def field_count(self, db: str, tb: str):
+        sql = f"select count(name) as `count` from system.columns where database='{db}' and table='{tb}'"
+        res = await self.execute(sql)
+        return res[0]['count']
+
+    async def distinct(self, db: str, tb: str, field: str, where: str = '1'):
+        sql = f'select distinct `{field}` as v from {db}.{tb} where {where}'
+        res = await self.query_dataframe(sql)
+        return res['v'].to_list()
+
+    async def yesterday_event_count(self, db: str):
+        today = datetime.date.today()
+        yesterday = today - datetime.timedelta(days=1)
+        today_str = today.strftime('%Y-%m-%d %H:%M:%S')
+        yesterday_str = yesterday.strftime('%Y-%m-%d %H:%M:%S')
+        sql = f"select `#event_name` as event_name, count() as v from {db}.event where `#event_time`>='{yesterday_str}' and `#event_time`<'{today_str}' group by `#event_name`"
+        df = await self.query_dataframe(sql)
+        return df.set_index('event_name').T.to_dict()
+
+    async def get_columns(self, db: str, tb: str):
+        sql = f"select name,type from system.columns where database='{db}' and table='{tb}'"
+        df = await self.query_dataframe(sql)
+        return df.T.to_dict().values()
+
+ckdb = CKDrive()
+
+
+def get_ck_db() -> CKDrive:
+    return ckdb

db/ckdb_utils.py

@@ -0,0 +1,15 @@
+from aioch import Client
+
+from core.config import settings
+from .ckdb import CKDrive
+
+
+async def connect_to_ck(pool_size=15):
+    for i in range(pool_size):
+        client = Client(**settings.CK_CONFIG)
+        CKDrive.ClientPool.add(client)
+
+
+async def close_ck_connection():
+    for c in CKDrive.ClientPool:
+        await c.disconnect()

db/mongodb.py

@@ -0,0 +1,14 @@
+from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorDatabase
+
+from core.config import settings
+
+
+class DataBase:
+    client: AsyncIOMotorClient = None
+
+
+db = DataBase()
+
+
+def get_database() -> AsyncIOMotorDatabase:
+    return db.client[settings.MDB_DB]

db/mongodb_utils.py

@@ -0,0 +1,12 @@
+from motor.motor_asyncio import AsyncIOMotorClient
+
+from core.config import settings
+from .mongodb import db
+
+
+def connect_to_mongo():
+    db.client = AsyncIOMotorClient(settings.DATABASE_URI)
+
+
+def close_mongo_connection():
+    db.client.close()

init_db.py

@@ -0,0 +1,86 @@
+import crud
+import schemas
+from core.config import settings
+
+# 创建一个超级用户、、
+from db import connect_to_mongo, get_database
+import asyncio
+
+connect_to_mongo()
+db = get_database()
+
+
+async def create_superuser():
+    user = await crud.user.get_by_user(db=db, name=settings.SUPERUSER_NAME)
+    if not user:
+        user_in = schemas.UserCreate(
+            name=settings.SUPERUSER_NAME,
+            email=settings.SUPERUSER_EMAIL,
+            password=settings.SUPERUSER_PASSWORD,
+            nickname=settings.SUPERUSER_NICKNAME,
+            is_superuser=True,
+        )
+        await crud.user.create(db, user_in)
+    await crud.user.create_index(db)
+
+
+async def project_index():
+    await crud.project.create_index(db)
+
+
+async def folder_index():
+    await crud.folder.create_index(db)
+
+
+async def space_index():
+    await crud.space.create_index(db)
+
+
+async def dashboard_index():
+    await crud.dashboard.create_index(db)
+
+
+async def report_index():
+    await crud.report.create_index(db)
+
+
+async def data_attr_index():
+    await crud.data_attr.create_index(db)
+
+
+async def event_mana():
+    await crud.event_mana.create_index(db)
+
+
+async def api_list_index():
+    await crud.api_list.create_index(db)
+
+async def role_index():
+    await crud.role.create_index(db)
+
+
+async def authority_init():
+    await crud.authority.create_index(db)
+    await crud.authority.create(db, 'p', '*', '*', '/docs', '*')
+    await crud.authority.create(db, 'p', '*', '*', '/openapi.json', '*')
+    await crud.authority.create(db, 'p', '*', '*', '/api/v1/user/login', '*')
+    await crud.authority.create(db, 'p', '*', '*', '/docs', '*')
+    await crud.authority.create(db, 'p', '*', '*', '/api/v1/project/', '*')
+
+
+async def main():
+    # await create_superuser()
+    # await project_index()
+    # await folder_index()
+    # await space_index()
+    # await dashboard_index()
+    # await report_index()
+    await authority_init()
+    # await data_attr_index()
+    # await event_mana()
+    # await api_list_index()
+    # await role_index()
+
+
+loop = asyncio.get_event_loop()
+loop.run_until_complete(main())

main.py

@@ -0,0 +1,165 @@
+import binascii
+import time
+
+import uvicorn
+from fastapi import FastAPI, Request
+from fastapi.exceptions import RequestValidationError
+from starlette.middleware.cors import CORSMiddleware
+from starlette.authentication import AuthenticationBackend, AuthenticationError, AuthCredentials, BaseUser, SimpleUser
+from starlette.middleware.authentication import AuthenticationMiddleware
+from starlette.requests import HTTPConnection
+from starlette.responses import Response, JSONResponse
+
+import crud
+import schemas
+
+
+from db import connect_to_mongo, close_mongo_connection, get_database
+from db.ckdb_utils import connect_to_ck, close_ck_connection
+from db.redisdb_utils import connect_to_redis, close_redis_connection
+from utils import *
+from api.api_v1.api import api_router
+from core.config import settings
+from api.deps import get_current_user2
+
+app = FastAPI(title=settings.PROJECT_NAME)
+app.include_router(api_router, prefix=settings.API_V1_STR)
+
+app.add_event_handler("startup", connect_to_mongo)
+app.add_event_handler("startup", connect_to_redis)
+app.add_event_handler("startup", connect_to_ck)
+
+app.add_event_handler("shutdown", close_mongo_connection)
+app.add_event_handler("shutdown", close_redis_connection)
+app.add_event_handler("shutdown", close_ck_connection)
+
+
+class CurrentUser(BaseUser):
+    def __init__(self, username: str, user_id: str) -> None:
+        self.username = username
+        self.id = user_id
+
+    @property
+    def is_authenticated(self) -> bool:
+        return True
+
+    @property
+    def display_name(self) -> str:
+        return self.username
+
+    @property
+    def identity(self) -> str:
+        return ''
+
+
+class BasicAuth(AuthenticationBackend):
+    async def authenticate(self, request):
+        if "Authorization" not in request.headers or request.scope.get('path') == '/api/v1/user/login':
+            return None
+
+        auth = request.headers["Authorization"]
+        if len(auth) < 20:
+            return None
+        try:
+            user = get_current_user2(auth.split(' ')[1])
+        except (ValueError, UnicodeDecodeError, binascii.Error):
+            raise AuthenticationError("身份验证失败，请重新登录")
+
+        return AuthCredentials(["authenticated"]), CurrentUser(user.name, user.id)
+
+
+def login_expired(conn: HTTPConnection, exc: Exception) -> Response:
+    return JSONResponse(schemas.Msg(code=-5, msg='请重新登录').dict(), status_code=200)
+#处理路由权限问题
+@app.middleware("http")
+async def panduan_quanxian_url(request: Request, call_next):
+    #user_id=request.user.id
+    #user=request.user.username
+    start_time = int(time.time() * 1000)
+    response = await call_next(request)
+    process_time = int(time.time() * 1000) - start_time
+    response.headers["X-Process-Time"] = str(process_time)
+    url=request.url.path
+    if 'docs' in url or 'openapi.json' in url:
+        return response
+    if url == '/api/v1/user/login':
+        return response
+    game=request.url.query.split('=')[1]
+    if 'undefined' in game:
+        return response
+    if '&' in game:
+        game=game.split('&')[0]
+    judge_url = await crud.user_url.get_quanxian(get_database(), schemas.Url_quanxian(user_id=request.user.id))
+    if judge_url == {}:
+        # data='没有匹配这个游戏'
+        return Response(schemas.Msg(code=0, msg='没有操作权限',data='').json())
+    if game not in judge_url['game']:
+        #data='没有匹配这个游戏'
+        return Response(schemas.Msg(code=0, msg='没有操作权限',data='' ).json())
+    quanxian_dict={}
+    for i in range(len(judge_url['game'])):
+        quanxian_dict[judge_url['game'][i]]=judge_url['quanxian'][i]
+    user_list=await crud.url_list.get_url(get_database(),schemas.Url_list(name=quanxian_dict[game]))
+    api_list=[]
+    state_list=[]
+    api_dict={}
+    for i in user_list:
+        for api in i['api_list']:
+            api_list.append(api)
+        for  quanxian in i['state']:
+            state_list.append(quanxian)
+    for i in range(len(api_list)):
+        api_dict[api_list[i]]=state_list[i]
+    if url not in api_list:
+        # data='没有对应路由'
+        return Response(schemas.Msg(code=0, msg='没有操作权限',data='').json())
+    elif api_dict[url] != True:
+        # data='路由为False'
+        return Response(schemas.Msg(code=0, msg='没有操作权限',data='').json())
+
+    return response
+
+
+app.add_middleware(AuthenticationMiddleware, backend=BasicAuth(), on_error=login_expired)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=['*'],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(request, exc):
+    return Response(schemas.Msg(code=-4, msg='请求错误', data=str(exc)).json(), status_code=400)
+
+
+@app.exception_handler(Exception)
+async def http_exception_handler(request, exc):
+    return Response(schemas.Msg(code=-3, msg='服务器错误').json(), status_code=500)
+
+
+@app.middleware("http")
+async def add_process_time_header(request: Request, call_next):
+    start_time = int(time.time() * 1000)
+    response = await call_next(request)
+    process_time = int(time.time() * 1000) - start_time
+    response.headers["X-Process-Time"] = str(process_time)
+    user_id = 'anonymous'
+    try:
+        user_id = request.user.id
+    except:
+        pass
+    await crud.api_log.insert_log(get_database(), schemas.ApiLogInsert(
+        api=str(request.url),
+        ms=process_time,
+        user_id=user_id
+    ))
+    return response
+
+
+if __name__ == '__main__':
+    uvicorn.run(app='main:app', host="10.0.0.240", port=7800, reload=True, debug=True)
+    #uvicorn.run(app='main:app', host="0.0.0.0", port=7800, reload=True, debug=True)

models/behavior_analysis.py

@@ -0,0 +1,890 @@
+import re
+from typing import Tuple
+
+import arrow
+import sqlalchemy as sa
+import json
+
+from fastapi import Depends
+
+import pandas as pd
+import numpy as np
+
+from sqlalchemy import func, or_, and_, not_
+
+import crud
+import schemas
+from core.config import settings
+from db import get_database
+
+from db.redisdb import get_redis_pool, RedisDrive
+from models.user_label import UserClusterDef
+
+
+class CombinationEvent:
+    def __init__(self, data, string, format):
+        self.data = data
+        self.string = string
+        self.pattern = re.compile('[+\-*/]')
+        self.format = format
+        self.events_name = []
+
+    def parse(self):
+        opts = self.pattern.findall(self.string)
+        factors = self.pattern.split(self.string)
+        result = pd.Series(self.data[int(factors[0])]['values'][0])
+        for i, opt in enumerate(opts):
+            b = pd.Series(self.data[int(factors[i + 1])]['values'][0])
+            result = settings.ARITHMETIC[opt](result, b).fillna(0)
+        if self.format == 'percent':
+            result = round(result * 100, 2)
+        elif self.format == 'float':
+            result = round(result, 2)
+        elif self.format == 'integer':
+            result = result.astype(int)
+        result.replace(np.inf, 0, inplace=True)
+        return result.to_list(), round(result.sum(), 2), round(result.mean(), 2)
+
+
+class CustomEvent:
+    def __init__(self, tbl, string, format):
+        self.tbl = tbl
+        self.string = string
+        self.pattern = re.compile('[+\-*/]')
+        self.format = format
+        self.events_name = []
+
+    def _parse(self, s):
+        m = s.split('.')
+        if len(m) == 3:
+            event_name, attr, comp = m
+            self.events_name.append(event_name)
+            return getattr(func, comp)(getattr(func, 'if')(getattr(self.tbl.c, '#event_name') == event_name,
+                                                           getattr(self.tbl.c, attr), 0))
+        elif len(m) == 2:
+            event_name, comp = m
+            self.events_name.append(event_name)
+            # 总次数
+            if comp == 'total_count':
+                return func.sum(getattr(func, 'if')(getattr(self.tbl.c, '#event_name') == event_name, 1, 0))
+            elif comp == 'touch_user_count':
+                return func.uniqCombined(getattr(func, 'if')(getattr(self.tbl.c, '#event_name') == event_name,
+                                                             getattr(self.tbl.c, '#account_id'), None))
+            elif comp == 'touch_user_avg':
+                return func.divide(
+                    func.sum(getattr(func, 'if')(getattr(self.tbl.c, '#event_name') == event_name, 1, 0)),
+                    func.uniqCombined(getattr(func, 'if')(getattr(self.tbl.c, '#event_name') == event_name,
+                                                          getattr(self.tbl.c, '#account_id'), None)))
+        elif len(m) == 1:
+            n = int(m[0])
+            return n
+
+    def str2obj(self, factors, opts):
+        sel = None
+        for i, factor in enumerate(factors):
+            if i == 0:
+                sel = self._parse(factor)
+            else:
+                tmp = self._parse(factor)
+                sel = settings.ARITHMETIC[opts[i - 1]](sel, tmp)
+        return sel
+
+    def parse(self):
+        factors = self.pattern.split(self.string)
+        opts = self.pattern.findall(self.string)
+        sel = self.str2obj(factors, opts)
+        decimal = 2
+        if self.format == 'percent':
+            sel = sel * 100
+        elif format == 'integer':
+            decimal = 0
+        elif format == 'float':
+            decimal = 2
+        sel = func.round(sel, decimal).label('values')
+        res = {
+            'event_name': self.events_name,
+            'select': sel
+        }
+        return res
+
+
+class BehaviorAnalysis:
+    def __init__(self, game: str, data_in: schemas.CkQuery, rdb: RedisDrive = Depends(get_redis_pool)):
+        self.game = game
+        self.rdb = rdb
+        self.user_tbl = None
+        self.event_tbl = None
+        self.data_in = data_in
+        self.event_view = dict()
+        self.events = [dict()]
+
+        self.zone_time: int = 0
+        self.start_date = None
+        self.end_date = None
+        self.global_filters = None
+        self.groupby = None
+        self.time_particle = None
+        self.date_range = None
+        self.unit_num = None
+        self.report_name = None
+        self.combination_event = []
+        self.ext_filters = (self.data_in.ext_filter.get('filts', []), self.data_in.ext_filter.get('relation', 'and'))
+        self.global_relation = 'and'
+        self.data_where = []
+
+    async def init(self, *args, **kwargs):
+
+        if self.data_in.report_id:
+            db = get_database()
+            report = await crud.report.get(db, id=self.data_in.report_id)
+            self.event_view = report['query']['eventView']
+            self.events = report['query']['events']
+            if self.event_view.get('date_type') == 'static':
+                pass
+            else:
+                try:
+                    e_days = self.event_view['e_days']
+                    s_days = self.event_view['s_days']
+                except:
+                    # 兼容以前的
+                    e_days, s_days = self.event_view['recentDay'].split('-')
+
+                self.event_view['endTime'] = arrow.get().shift(days=-int(e_days)).strftime('%Y-%m-%d 23:59:59')
+                self.event_view['startTime'] = arrow.get().shift(days=-int(s_days)).strftime('%Y-%m-%d 00:00:00')
+
+            self.event_view['startTime'] = self.data_in.ext_filter.get('startTime') or self.event_view['startTime']
+            self.event_view['endTime'] = self.data_in.ext_filter.get('endTime') or self.event_view['endTime']
+
+            self.report_name = report["name"]
+
+
+        else:
+            self.event_view = self.data_in.eventView
+            self.events = self.data_in.events
+
+        await self._init_table()
+        self.zone_time = self._get_zone_time()
+        self.time_particle = self._get_time_particle_size()
+        self.start_date, self.end_date, self.date_range = self._get_date_range()
+        self.global_filters = self._get_global_filters()
+        self.groupby = self._get_group_by()
+        self.unit_num = self._get_unit_num()
+        self.global_relation = self.event_view.get('relation', 'and')
+
+        # 用户自带过滤
+        if 'data_where' in kwargs:
+            self.data_where = kwargs['data_where'].get(self.game, [])
+            self.global_filters.extend(self.data_where)
+        # self.global_filters.extend(self.data_in.ext_filter.get('filts', []))
+
+    def _get_time_particle_size(self):
+        return self.event_view.get('timeParticleSize') or 'P1D'
+
+    def _get_unit_num(self):
+        return self.event_view.get('unitNum')
+
+    def _get_group_by(self):
+
+        return [getattr(self.event_tbl.c, item['columnName']) for item in self.event_view.get('groupBy', [])]
+
+    def _get_zone_time(self):
+        return int(self.event_view.get('zone_time', 8))
+
+    def _get_date_range(self) -> Tuple[str, str, list]:
+        start_date: str = self.event_view.get('startTime')
+        end_date: str = self.event_view.get('endTime')
+        if self.time_particle == 'HOUR':
+            date_range = [i for i in range(24)]
+            return start_date, end_date, date_range
+
+        date_range = pd.date_range(start_date, end_date, freq=settings.PROPHET_TIME_GRAIN_MAP[self.time_particle],
+                                   tz='UTC').tolist()
+        if self.time_particle in ('P1D', 'P1W', 'P1M'):
+            date_range = [item.date() for item in date_range]
+            # start_date = date_range[0].strftime('%Y-%m-%d')
+            # end_date = date_range[-1].strftime('%Y-%m-%d')
+
+        return start_date, end_date, date_range
+
+    def _get_global_filters(self):
+        return self.event_view.get('filts') or []
+
+    async def _init_table(self):
+        """
+        从redis中取出表字段，构建表结构
+        :return:
+        """
+        res_json = await self.rdb.get(f'{self.game}_user')
+        columns = json.loads(res_json).keys()
+        metadata = sa.MetaData(schema=self.game)
+        self.user_tbl = sa.Table('user_view', metadata, *[sa.Column(column) for column in columns])
+
+        res_json = await self.rdb.get(f'{self.game}_event')
+        columns = json.loads(res_json).keys()
+        metadata = sa.MetaData(schema=self.game)
+        # self.event_tbl = sa.Table('event_view', metadata, *[sa.Column(column) for column in columns])
+        self.event_tbl = sa.Table('event', metadata, *[sa.Column(column) for column in columns])
+
+    async def handler_filts(self, *filters):
+        """
+
+        :param filters: (filts:list,relation:str)
+        :param g_f:
+        :param relation:
+        :return:
+        """
+
+        user_filters = []
+        event_filters = []
+        for filter in filters:
+            filts = filter[0]
+            relation = filter[1]
+            user_filter = []
+            event_filter = []
+            for item in filts:
+                comparator = item['comparator']
+                if item['tableType'] == 'user':
+                    where = user_filter
+                elif item['tableType'] == 'event':
+                    where = event_filter
+                elif item['tableType'] == 'user_label':
+                    user_cluster_def = UserClusterDef(self.game, item['columnName'], self.data_where)
+                    await user_cluster_def.init()
+                    sub_qry = user_cluster_def.to_sql_qry()
+                    if comparator == 'in':
+                        event_filter.append(sa.Column('#account_id').in_(sub_qry))
+                    else:
+                        event_filter.append(sa.Column('#account_id').notin_(sub_qry))
+
+                    continue
+                else:
+                    continue
+
+                tbl = getattr(self, f'{item["tableType"]}_tbl')
+                col = getattr(tbl.c, item['columnName'])
+                # 日期类型处理时区
+                if item.get('data_type') == 'datetime':
+                    col = func.addHours(col, self.zone_time)
+
+                ftv = item['ftv']
+                if comparator == '==':
+                    if len(ftv) > 1:
+                        where.append(or_(*[col == v for v in ftv]))
+                    else:
+                        where.append(col == ftv[0])
+                elif comparator == '>=':
+                    where.append(col >= ftv[0])
+                elif comparator == '<=':
+                    where.append(col <= ftv[0])
+                elif comparator == '>':
+                    where.append(col > ftv[0])
+                elif comparator == '<':
+                    where.append(col < ftv[0])
+
+                elif comparator == 'is not null':
+                    where.append(col.isnot(None))
+                elif comparator == 'is null':
+                    where.append(col.is_(None))
+
+                elif comparator == 'like':
+                    where.append(col.like(f'%{ftv[0]}%'))
+
+                elif comparator == 'not like':
+                    where.append(col.notlike(f'%{ftv[0]}%'))
+
+                elif comparator == 'in':
+                    where.append(col.in_(ftv))
+
+                elif comparator == '!=':
+                    where.append(col != ftv[0])
+            if relation == 'and':
+                if event_filter:
+                    event_filters.append(and_(*event_filter))
+                if user_filter:
+                    user_filters.append(and_(*user_filter)),
+            else:
+                if event_filter:
+                    event_filters.append(or_(*event_filter))
+                if user_filter:
+                    user_filters.append(or_(*user_filter))
+
+        return event_filters, user_filters
+
+    async def retention_model_sql(self):
+        event_name_a = self.events[0]['eventName']
+        event_name_b = self.events[1]['eventName']
+        visit_name = self.events[0].get('event_attr_id')
+        event_time_col = getattr(self.event_tbl.c, '#event_time')
+        event_name_col = getattr(self.event_tbl.c, '#event_name')
+        e_account_id_col = getattr(self.event_tbl.c, '#account_id')
+        u_account_id_col = getattr(self.user_tbl.c, '#account_id')
+        date_col = sa.Column('date')
+        who_visit = e_account_id_col
+        if visit_name:
+            who_visit = getattr(self.event_tbl.c, visit_name)
+
+        filters, _ = await self.handler_filts((self.events[0]['filts'], self.events[0].get('relation')),
+                                              self.ext_filters)
+        filters = filters or [1]
+        selectd = [func.toStartOfDay(func.addHours(event_time_col, self.zone_time)).label('date'),
+                   *self.groupby,
+                   func.arrayDistinct(
+                       (func.groupArray(
+                           func.if_(func.and_(event_name_col == event_name_a, *filters), who_visit, None)))).label(
+                       'val_a'),
+
+                   func.length(sa.Column('val_a')).label('amount_a'),
+                   func.length(sa.Column('val_b')).label('amount_b'),
+                   ]
+
+        if event_name_b == '*':
+            val_b = func.arrayDistinct(
+                (func.groupArray(func.if_(1, who_visit, None)))).label('val_b'),
+            selectd.insert(-2, *val_b)
+        else:
+            val_b = func.arrayDistinct(
+                (func.groupArray(func.if_(event_name_col == event_name_b, who_visit, None)))).label('val_b'),
+            selectd.insert(-2, *val_b)
+
+        base_where = [
+            func.addHours(event_time_col, self.zone_time) >= self.start_date,
+            func.addHours(event_time_col, self.zone_time) <= self.end_date,
+        ]
+
+        event_filter, user_filter = await self.handler_filts(
+            (self.global_filters, self.global_relation),
+            self.ext_filters
+        )
+
+        groupby = [date_col] + self.groupby
+        oredrby = [date_col]
+        if user_filter:
+            qry = sa.select(selectd).select_from(
+                self.event_tbl.join(self.user_tbl, u_account_id_col == e_account_id_col)).where(
+                and_(*user_filter, *event_filter, *base_where)).group_by(*groupby).order_by(
+                *oredrby).limit(10000)
+        else:
+            qry = sa.select(selectd).where(and_(*base_where, *event_filter)).group_by(*groupby).order_by(
+                *oredrby).limit(10000)
+        sql = str(qry.compile(compile_kwargs={"literal_binds": True}))
+        print(sql)
+        return {'sql': sql,
+                'groupby': ['date'] + [i.key for i in self.groupby],
+                'date_range': self.date_range,
+                'event_name': [event_name_a, event_name_b],
+                'unit_num': self.unit_num,
+                'time_particle': self.time_particle,
+                'start_date': self.start_date[:10],
+                'end_date': self.end_date[:10],
+                }
+
+    async def event_model_sql(self):
+        sqls = []
+        event_time_col = getattr(self.event_tbl.c, '#event_time')
+        for event in self.events:
+            operator_ = event.get('operator_val','')
+            #排头显示名
+            event_name_display = event.get('eventNameDisplay')
+            is_show = event.get('is_show', True)
+
+            select_exprs = []
+            if self.time_particle != 'total':
+                select_exprs.append(
+                    settings.TIME_GRAIN_EXPRESSIONS[self.time_particle](event_time_col, self.zone_time))
+
+            base_where = [
+                func.addHours(event_time_col, self.zone_time) >= self.start_date,
+                func.addHours(event_time_col, self.zone_time) <= self.end_date,
+            ]
+            event_name_col = getattr(self.event_tbl.c, '#event_name')
+            format = event.get('format') or 'float'
+
+            # 兼容以前的结构
+            if event.get('customEvent'):
+                event['customType'] = event.get('customType') or 'formula'
+
+            if event.get('customType') == 'formula':
+                if event.get('customEvent'):
+                    #组合公式的内容
+                    formula = event.get('customEvent')
+                    custom = CustomEvent(self.event_tbl, formula, format).parse()
+                    event_name = custom['event_name']
+                    where = [event_name_col.in_(event_name)]
+                    event_filter, _ = await self.handler_filts((event['filts'], event.get('relation')),
+                                                               (self.global_filters, self.global_relation),
+                                                               self.ext_filters
+                                                               )
+                    select_exprs.extend(self.groupby)
+                    qry = sa.select(
+                        *select_exprs,
+                        custom['select']
+                    ).where(*base_where, *where, *event_filter)
+
+            # 指标组合计算
+            elif event.get('customType') == 'combination':
+                sqls.append({'combination_event': event.get('customEvent'),
+                             'time_particle': self.time_particle,
+                             'start_date': self.start_date[:10],
+                             'end_date': self.end_date[:10],
+                             'event_name': event.get('eventNameDisplay'),
+                             'format': event.get('format') or 'float',
+                             'date_range': self.date_range,
+                             'is_show': is_show,
+                             }
+                            )
+                continue
+            else:
+                event_name = event['event_name']
+
+                select_exprs += self.groupby
+                if event_name != '*':
+                    base_where.append(event_name_col == event_name)
+
+                analysis = event['analysis']
+                event_filter, user_filter = await self.handler_filts(
+                    (event['filts'], event.get('relation', 'and')),
+                    (self.global_filters, self.global_relation)
+                    , self.ext_filters
+                )
+
+                u_account_id_col = getattr(self.user_tbl.c, '#account_id')
+                # 按账号聚合
+                e_account_id_col = getattr(self.event_tbl.c, '#account_id')
+
+                if operator_ == '':
+                    # 聚合方式
+                    if analysis == 'total_count':
+                        selectd = select_exprs + [func.count().label('values')]
+                    elif analysis == 'touch_user_count':
+                        selectd = select_exprs + [func.count(sa.distinct(e_account_id_col)).label('values')]
+                    elif analysis == 'touch_user_avg':
+                        selectd = select_exprs + [
+                            func.round((func.count() / func.count(sa.distinct(e_account_id_col))), 2).label(
+                                'values')]
+                    else:
+                        selectd = select_exprs + [
+                            func.round(getattr(func, analysis)(getattr(self.event_tbl.c, event['event_attr_id'])), 2).label(
+                                'values')]
+                else:
+                    operator_val=int(operator_)
+                    operator=event['operator']  #运算符号
+                    if analysis == 'total_count':
+                        selectd = select_exprs + [settings.ARITHMETIC[operator](func.count(),operator_val).label('values')]
+                    elif analysis == 'touch_user_count':
+                        selectd = select_exprs + [settings.ARITHMETIC[operator](func.count(sa.distinct(e_account_id_col)),operator_val).label('values')]
+                    elif analysis == 'touch_user_avg':
+                        selectd = select_exprs + [
+                            settings.ARITHMETIC[operator](func.round((func.count() / func.count(sa.distinct(e_account_id_col))), 2),operator_val).label(
+                                'values')]
+                    else:
+                        selectd = select_exprs + [
+                            settings.ARITHMETIC[operator](func.round(getattr(func, analysis)(getattr(self.event_tbl.c, event['event_attr_id'])), 2),operator_val).label(
+                                'values')]
+
+                if user_filter:
+                    qry = sa.select(selectd).select_from(
+                        self.event_tbl.join(self.user_tbl, u_account_id_col == e_account_id_col)).where(
+                        and_(*user_filter, *event_filter, *base_where))
+
+                else:
+                    qry = sa.select(selectd).where(and_(*event_filter, *base_where))
+
+            qry = qry.group_by(*select_exprs)
+            if self.time_particle != 'total':
+                qry = qry.order_by(sa.Column('date'))
+            else:
+                qry = qry.order_by(sa.Column('values').desc())
+            qry = qry.limit(10000)
+
+            sql = str(qry.compile(compile_kwargs={"literal_binds": True}))
+            print(sql)
+            # 单独付费率的拿出来
+            if event.get('customEvent') == 'pay.touch_user_count/login.touch_user_count':
+                stat_date=self.start_date
+                end_date=self.end_date
+                game=self.game
+                sql=f"""
+                 select aa.date as date,round((a/b)*100,2) as values  from
+ (select toDate(addHours({game}.event."#event_time", 8)) AS date,uniqCombined(if({game}.event."#event_name" = 'pay', {game}.event."#account_id", NULL)) as a from {game}.event
+ WHERE addHours({game}.event."#event_time", 8) >= '{stat_date}' AND addHours({game}.event."#event_time", 8) <= '{end_date}'
+ AND {game}.event."#event_name"='pay' and orderid NOT LIKE '%GM%'  GROUP BY toDate(addHours({game}.event."#event_time", 8))) as aa
+ LEFT join
+ (SELECT toDate(addHours({game}.event."#event_time", 8)) AS date, round(uniqExact({game}.event."#account_id"), 2) AS b 
+FROM {game}.event 
+WHERE addHours({game}.event."#event_time", 8) >= '{stat_date}' AND addHours({game}.event."#event_time", 8) <= '{end_date}' 
+GROUP BY toDate(addHours({game}.event."#event_time", 8))) as bb on aa.date = bb.date ORDER by date
+                """
+            # 单独把新增付费人数（以设备为维度）拿出来
+            if event.get('event_attr') == '触发用户数' and ['is_new_device', 'orderid']== [i['columnName'] for i in event.get('filts')]:
+                stat_date=self.start_date
+                end_date=self.end_date
+                game=self.game
+                sql=f"""SELECT toDate(addHours("#event_time", 8)) as date,
+round(uniqExact("#distinct_id"), 2) AS values FROM
+(SELECT toDate(addHours("#event_time", 8)) as date,"#event_time",`#event_name`,`#distinct_id`,`#account_id` from {game}.event WHERE 
+addHours("#event_time", 8) >= '{stat_date}' AND addHours("#event_time", 8) <= '{end_date}'
+and `#event_name` = 'pay' and orderid NOT LIKE '%GM%') a
+inner join
+(SELECT toDate(addHours("#event_time", 8)) as date,"#event_time",is_new_device,`#distinct_id`,`#event_name`,`#account_id` from {game}.event WHERE 
+addHours("#event_time", 8) >= '{stat_date}' AND addHours("#event_time", 8) <= '{end_date}' and
+`#event_name` = 'create_account' and is_new_device = 1) b on a.`#distinct_id`= b.`#distinct_id` and a.date = b.date
+GROUP BY toDate(addHours("#event_time", 8))"""
+
+            sqls.append({'sql': sql,
+                         'groupby': [i.key for i in self.groupby],
+                         'date_range': self.date_range,
+                         'event_name': event_name_display or event_name,
+                         'format': format,
+                         'report_name': self.report_name or 'temp',
+                         'time_particle': self.time_particle,
+                         'start_date': self.start_date[:10],
+                         'end_date': self.end_date[:10],
+                         'is_show': is_show,
+                         })
+
+        return sqls
+    #在漏斗分析，事件分析模型里面都有用到这块
+    async def funnel_model_sql(self):
+        """
+SELECT level, count(*) AS values
+FROM (SELECT windowFunnel(86400)(shjy.event."#event_time", shjy.event."#event_name" = 'create_role',
+                          shjy.event."#event_name" = 'login') AS level
+      FROM shjy.event
+      WHERE addHours(shjy.event."#event_time", 8) >= '2021-05-16 00:00:00'
+        AND addHours(shjy.event."#event_time", 8) <= '2021-06-14 23:59:59'
+      GROUP BY shjy.event."#account_id") AS anon_1
+GROUP BY level
+ORDER BY level
+        :return:
+        """
+
+        windows_gap = self.event_view['windows_gap'] * 86400
+        event_time_col = getattr(self.event_tbl.c, '#event_time')
+        event_name_col = getattr(self.event_tbl.c, '#event_name')
+        date_col = func.toStartOfDay(func.addHours(event_time_col, self.zone_time)).label('date')
+        e_account_id_col = getattr(self.event_tbl.c, '#account_id')
+
+        sub_group = [date_col, *self.groupby, e_account_id_col]
+        conds = []
+        cond_level = []
+        for item in self.events:
+            event_filter, _ = await self.handler_filts((item['filts'], item.get('relation', 'and'))
+                                                       , self.ext_filters)
+            conds.append(
+                and_(event_name_col == item['eventName'], *event_filter)
+            )
+            cond_level.append(item['eventName'])
+        #     todo 替换 _windows_gap_
+        subq = sa.select(*[sa.Column(i.key) for i in self.groupby], date_col,
+                         func.windowFunnel_windows_gap__(event_time_col, *conds).label('level')).select_from(
+            self.event_tbl)
+
+        g_event_filter, _ = await self.handler_filts((self.global_filters, self.global_relation)
+                                                     , self.ext_filters)
+        where = [
+            func.addHours(event_time_col, self.zone_time) >= self.start_date,
+            func.addHours(event_time_col, self.zone_time) <= self.end_date,
+            *g_event_filter
+        ]
+        subq = subq.where(and_(*where)).group_by(*sub_group)
+        subq = subq.subquery()
+
+        qry = sa.select(sa.Column('date'), *[sa.Column(i.key) for i in self.groupby], sa.Column('level'),
+                        func.count().label('values')).select_from(subq) \
+            .where(sa.Column('level') > 0) \
+            .group_by(sa.Column('date'), *[sa.Column(i.key) for i in self.groupby], sa.Column('level')) \
+            .order_by(sa.Column('date'), *[sa.Column(i.key) for i in self.groupby], sa.Column('level'))
+        sql = str(qry.compile(compile_kwargs={"literal_binds": True}))
+        # sql = sql.replace('_windows_gap_', f"({windows_gap},'strict_increase')")
+        sql = sql.replace('_windows_gap_', f"({windows_gap})")
+        print(sql)
+        return {'sql': sql,
+                'groupby': [i.key for i in self.groupby],
+                'date_range': self.date_range,
+                'cond_level': cond_level,
+                'time_particle': self.time_particle,
+                'start_date': self.start_date[:10],
+                'end_date': self.end_date[:10],
+                }
+
+    async def scatter_model_sql(self):
+        event = self.events[0]
+        event_name = event['eventName']
+        analysis = event['analysis']
+        if analysis in ['list_distinct',"set_distinct","ele_distinct"]:
+            analysis = 'max'
+        e_account_id_col = getattr(self.event_tbl.c, '#account_id').label('uid')
+        u_account_id_col = getattr(self.user_tbl.c, '#account_id')
+        event_name_col = getattr(self.event_tbl.c, '#event_name')
+        event_time_col = getattr(self.event_tbl.c, '#event_time').label('date')
+        event_date_col = settings.TIME_GRAIN_EXPRESSIONS[self.time_particle](event_time_col, self.zone_time)
+
+        quota_interval_arr = event.get('quotaIntervalArr')
+
+        where = [
+            # event_date_col >= self.start_date,
+            # event_date_col <= self.end_date,
+            func.addHours(event_time_col, self.zone_time) >= self.start_date,
+            func.addHours(event_time_col, self.zone_time) <= self.end_date,
+
+        ]
+        if event_name != '*':
+            where.append(event_name_col == event_name)
+        event_filter, user_filter = await self.handler_filts((event['filts'], event.get('relation', 'and')),
+                                                             (self.global_filters, self.global_relation)
+                                                             , self.ext_filters)
+        if user_filter:
+            where.append(e_account_id_col.in_(sa.select(u_account_id_col).where(*user_filter)))
+        where.extend(event_filter)
+        values_col = func.count().label('values')
+        if analysis in ['number_of_days', 'number_of_hours']:
+            values_col = func.count(func.distinct(e_account_id_col)).label('values')
+
+        if analysis in ['times', 'number_of_days', 'number_of_hours']:
+            if self.time_particle == 'total':
+                qry = sa.select(*self.groupby, values_col) \
+                    .where(and_(*where)) \
+                    .group_by(*self.groupby, e_account_id_col)
+            else:
+                qry = sa.select(event_date_col, *self.groupby, values_col) \
+                    .where(and_(*where)) \
+                    .group_by(event_date_col, *self.groupby, e_account_id_col)
+
+            sql = str(qry.compile(compile_kwargs={"literal_binds": True}))
+            print(sql)
+            return {
+                'sql': sql,
+                'interval_type': event['intervalType'],
+                'analysis': analysis,
+                'quota_interval_arr': quota_interval_arr,
+                'groupby': [i.key for i in self.groupby],
+                'time_particle': self.time_particle,
+                'start_date': self.start_date[:10],
+                'end_date': self.end_date[:10],
+            }
+        elif event.get('quota'):
+            event_attr_col = getattr(self.event_tbl.c, event['quota'])
+            if self.time_particle == 'total':
+                qry = sa.select(e_account_id_col,
+                                settings.CK_FUNC[analysis](event_attr_col).label('values')) \
+                    .where(and_(*where)) \
+                    .group_by(*self.groupby, e_account_id_col)
+            else:
+                # qry = sa.select(event_date_col, e_account_id_col,
+                #                 settings.CK_FUNC[analysis](event_attr_col).label('values')) \
+                #     .where(and_(*where)) \
+                #     .group_by(event_date_col, *self.groupby, e_account_id_col)
+                qry = sa.select(event_date_col, e_account_id_col,
+                                settings.CK_FUNC[analysis](event_attr_col).label('values')) \
+                    .where(and_(*where)) \
+                    .group_by(event_date_col,e_account_id_col)
+            sql = str(qry.compile(compile_kwargs={"literal_binds": True}))
+            print(sql)
+            return {
+                'sql': sql,
+                'interval_type': event['intervalType'],
+                'analysis': analysis,
+                'quota_interval_arr': quota_interval_arr,
+                'groupby': [i.key for i in self.groupby],
+                'time_particle': self.time_particle,
+                'start_date': self.start_date[:10],
+                'end_date': self.end_date[:10],
+            }
+
+    def trace_model_sql(self):
+        session_interval = self.event_view.get('session_interval')
+        session_type = self.event_view.get('session_type')
+        session_type_map = {
+            'minute': 60,
+            'second': 1,
+            'hour': 3600
+
+        }
+        interval_ts = session_interval * session_type_map.get(session_type, 60)
+        event_names = self.events.get('event_names')
+        source_event = self.events.get('source_event', {}).get('eventName')
+        source_type = self.events.get('source_event', {}).get('source_type')
+
+        sql_a = f"""with
+    '{source_event}' as start_event,
+    {tuple(event_names)} as evnet_all,
+    '{self.start_date}' as start_data,
+    '{self.end_date}' as end_data
+select event_chain,
+       count()                  as values
+from (with
+          toUInt32(minIf(`#event_time`, `#event_name` = start_event)) AS start_event_ts,
+          arraySort(
+                  x ->
+                      x.1,
+                  arrayFilter(
+                          x -> x.1 >= start_event_ts,
+                          groupArray((toUInt32(`#event_time`), `#event_name`))
+                      )
+              ) AS sorted_events,
+          arrayEnumerate(sorted_events) AS event_idxs,
+          arrayFilter(
+                  (x, y, z) -> z.1 >= start_event_ts and ((z.2 = start_event and y > {interval_ts}) or y > {interval_ts}) ,
+                  event_idxs,
+                  arrayDifference(sorted_events.1),
+                  sorted_events
+              ) AS gap_idxs,
+          arrayMap(x -> x, gap_idxs) AS gap_idxs_,
+          arrayMap(x -> if(has(gap_idxs_, x), 1, 0), event_idxs) AS gap_masks,
+          arraySplit((x, y) -> y, sorted_events, gap_masks) AS split_events
+      select `#account_id`,
+             arrayJoin(split_events)         AS event_chain_,
+             arrayMap(x ->
+                          x.2, event_chain_) AS event_chain,
+             has(event_chain, start_event)   AS has_midway_hit
+
+      from (select `#event_time`, `#event_name`, `#account_id`
+            from {self.game}.event
+            where addHours(`#event_time`, {self.zone_time}) >= start_data
+              and addHours(`#event_time`, {self.zone_time}) <= end_data
+              and `#event_name` in evnet_all)
+      group by `#account_id`
+      HAVING has_midway_hit = 1
+         )
+where arrayElement(event_chain, 1) = start_event
+GROUP BY event_chain
+ORDER BY values desc
+"""
+        sql_b = f"""with
+    '{source_event}' as end_event,
+    {tuple(event_names)} as evnet_all,
+    '{self.start_date}' as start_data,
+    '{self.end_date}' as end_data
+select event_chain,
+       count()                  as values
+from (with
+          toUInt32(maxIf(`#event_time`, `#event_name` = end_event)) AS end_event_ts,
+          arraySort(
+                  x ->
+                      x.1,
+                  arrayFilter(
+                          x -> x.1 <= end_event_ts,
+                          groupArray((toUInt32(`#event_time`), `#event_name`))
+                      )
+              ) AS sorted_events,
+          arrayEnumerate(sorted_events) AS event_idxs,
+          arrayFilter(
+                  (x, y, z) -> z.1 <= end_event_ts and (z.2 = end_event and y>{interval_ts}) OR y > {interval_ts},
+                  event_idxs,
+                  arrayDifference(sorted_events.1),
+                  sorted_events
+              ) AS gap_idxs,
+          arrayMap(x -> x+1, gap_idxs) AS gap_idxs_,
+          arrayMap(x -> if(has(gap_idxs_, x), 1,0), event_idxs) AS gap_masks,
+          arraySplit((x, y) -> y, sorted_events, gap_masks) AS split_events
+      select `#account_id`,
+             arrayJoin(split_events)         AS event_chain_,
+             arrayMap(x ->
+                          x.2, event_chain_) AS event_chain,
+             has(event_chain, end_event)   AS has_midway_hit
+      from (select `#event_time`, `#event_name`, `#account_id`
+            from {self.game}.event
+            where addHours(`#event_time`, {self.zone_time}) >= start_data
+              and addHours(`#event_time`, {self.zone_time}) <= end_data
+              and `#event_name` in evnet_all)
+      group by `#account_id`
+      HAVING has_midway_hit = 1
+         )
+where arrayElement(event_chain, -1) = end_event
+GROUP BY event_chain
+ORDER BY values desc"""
+
+        sql = sql_a if source_type == 'initial_event' else sql_b
+        print(sql)
+        return {
+            'sql': sql,
+            'time_particle': self.time_particle,
+            'start_date': self.start_date[:10],
+            'end_date': self.end_date[:10],
+        }
+
+    async def retention_model_sql2(self):
+        filter_item_type = self.event_view.get('filter_item_type')
+        filter_item = self.event_view.get('filter_item')
+        event_name_a = self.events[0]['eventName']
+        event_name_b = self.events[1]['eventName']
+
+        visit_name = self.events[0].get('event_attr_id')
+
+        where, _ = await self.handler_filts((self.events[0]['filts'], self.events[0].get('relation', 'and')),
+                                            (self.global_filters, self.global_relation)
+                                            , self.ext_filters)
+        where_a = '1'
+        if where:
+            qry = sa.select().where(*where)
+            sql = str(qry.compile(compile_kwargs={"literal_binds": True}))
+            where_a = 'WHERE '.join(sql.split('WHERE ')[1:])
+
+        where, _ = await self.handler_filts((self.events[1]['filts'], self.events[1].get('relation', 'and')),
+                                            (self.global_filters, self.global_relation)
+                                            , self.ext_filters)
+        where_b = '1'
+        if where:
+            qry = sa.select().where(*where)
+            sql = str(qry.compile(compile_kwargs={"literal_binds": True}))
+            where_b = sql.split('WHERE ')[1]
+
+        # 任意事件
+        event_name_b = 1 if event_name_b == '*' else f"`#event_name` = '{event_name_b}'"
+
+        days = (arrow.get(self.end_date).date() - arrow.get(self.start_date).date()).days
+        keep = []
+        cnt = []
+        retention_n = [*[k for k in range(1, 60)], 70-1, 75-1, 80-1, 85-1, 90-1, 95-1, 100-1, 110-1, 120-1, 150-1, 180-1, 210-1, 240-1, 270-1, 300-1,
+                       360-1]
+
+        """
+               cnt0-cnt1 as on1,
+       round(on1 * 100 / cnt0, 2)   as `0p1`,
+        """
+
+        for i in retention_n:
+            keep.append(
+                f"""cnt{i},
+                round(cnt{i} * 100 / cnt0, 2) as `p{i}`,
+                cnt0-cnt{i} as on{i},
+                round(on{i} * 100 / cnt0, 2)   as `op{i}`
+                """)
+            cnt.append(f"""sum(if(dateDiff('day',a.reg_date,b.visit_date)={i},1,0)) as cnt{i}""")
+        keep_str = ','.join(keep)
+        cnt_str = ','.join(cnt)
+
+        sql = f"""
+with '{event_name_a}' as start_event,
+    {event_name_b} as retuen_visit,
+    `{visit_name}` as visit,
+    '{self.start_date}' as start_data,
+    '{self.end_date}' as end_data,
+    toDate(addHours(`#event_time`, {self.zone_time})) as date
+
+select reg_date,
+       cnt0 ,
+       {keep_str}
+
+       from(select date, uniqExact(visit) as cnt0 from {self.game}.event
+where `#event_name` = start_event and addHours(`#event_time`, {self.zone_time}) >= start_data  and addHours(`#event_time`, {self.zone_time}) <= end_data and {where_a}
+group by date) reg left join
+(select a.reg_date,
+       {cnt_str}
+from (select date as reg_date, visit from {self.game}.event where `#event_name` = start_event and addHours(`#event_time`, {self.zone_time}) >= start_data  and addHours(`#event_time`, {self.zone_time}) <= end_data and {where_a} group by reg_date, visit) a
+         left join (select date as visit_date, visit from {self.game}.event where retuen_visit and addHours(`#event_time`, {self.zone_time}) >= start_data group by visit_date, visit) b on
+a.visit = b.visit
+group by a.reg_date) log on reg.date=log.reg_date
+"""
+        print(sql)
+        return {
+            'sql': sql,
+            'date_range': self.date_range,
+            'unit_num': self.unit_num,
+            'retention_n': retention_n,
+            'filter_item_type': filter_item_type,
+            'filter_item': filter_item,
+            'time_particle': self.time_particle,
+            'start_date': self.start_date[:10],
+            'end_date': self.end_date[:10],
+        }

models/user_analysis.py

@@ -0,0 +1,223 @@
+from typing import Tuple
+
+import arrow
+import sqlalchemy as sa
+import json
+
+from fastapi import Depends
+
+import pandas as pd
+
+from sqlalchemy import func, or_, and_, not_
+
+import crud
+import schemas
+from core.config import settings
+from db import get_database
+from db.redisdb import get_redis_pool, RedisDrive
+
+
+class UserAnalysis:
+    def __init__(self, game: str, data_in: schemas.CkQuery, rdb: RedisDrive = Depends(get_redis_pool)):
+        self.game = game
+        self.rdb = rdb
+        self.user_tbl = None
+        self.event_view = data_in.eventView
+        self.events = data_in.events
+
+        self.zone_time: int = 0
+        self.data_in = data_in
+
+        self.global_filters = []
+        self.groupby = None
+        self.time_particle = None
+        self.date_range = None
+        self.unit_num = None
+        self.global_relation = 'and'
+        self.ext_filters = (self.data_in.ext_filter.get('filts', []), self.data_in.ext_filter.get('relation', 'and'))
+
+    async def init(self, *args, **kwargs):
+        if self.data_in.report_id:
+            db = get_database()
+            report = await crud.report.get(db, id=self.data_in.report_id)
+            self.event_view = report['query']['eventView']
+            self.events = report['query']['events']
+
+        else:
+            self.event_view = self.data_in.eventView
+            self.events = self.data_in.events
+
+        await self._init_table()
+        self.zone_time = self._get_zone_time()
+        self.time_particle = self._get_time_particle_size()
+        self.groupby = self._get_group_by()
+        self.unit_num = self._get_unit_num()
+        self.global_relation = self.event_view.get('relation', 'and')
+        # 用户自带过滤
+        if 'data_where' in kwargs:
+            self.global_filters.extend(kwargs['data_where'].get(self.game, []))
+
+    async def _init_table(self):
+        """
+        从redis中取出表字段，构建表结构
+        :return:
+        """
+        res_json = await self.rdb.get(f'{self.game}_user')
+        columns = json.loads(res_json).keys()
+        metadata = sa.MetaData(schema=self.game)
+        self.user_tbl = sa.Table('user_view', metadata, *[sa.Column(column) for column in columns])
+
+    def _get_time_particle_size(self):
+        return self.event_view.get('timeParticleSize') or 'P1D'
+
+    def _get_unit_num(self):
+        return self.event_view.get('unitNum')
+
+    def _get_group_by(self):
+        return [getattr(self.user_tbl.c, item['columnName']) for item in self.event_view.get('groupBy', [])]
+
+    def _get_zone_time(self):
+        return int(self.event_view.get('zone_time', 8))
+
+    # def _get_filters(self, filters):
+    #     tbl = self.user_tbl
+    #     where = []
+    #     for item in filters:
+    #         col = getattr(tbl.c, item['columnName'])
+    #
+    #         comparator = item['comparator']
+    #         ftv = item['ftv']
+    #         if comparator == '==':
+    #             if len(ftv) > 1:
+    #                 where.append(or_(*[col == v for v in ftv]))
+    #             else:
+    #                 where.append(col == ftv[0])
+    #         elif comparator == '>=':
+    #             where.append(col >= ftv[0])
+    #         elif comparator == '<=':
+    #             where.append(col <= ftv[0])
+    #         elif comparator == '>':
+    #             where.append(col > ftv[0])
+    #         elif comparator == '<':
+    #             where.append(col < ftv[0])
+    #
+    #         elif comparator == 'is not null':
+    #             where.append(col.isnot(None))
+    #         elif comparator == 'is null':
+    #             where.append(col.is_(None))
+    #
+    #         elif comparator == '!=':
+    #             where.append(col != ftv[0])
+    #
+    #         elif comparator == 'like':
+    #             where.append(col.like(f'%{ftv[0]}%'))
+    #
+    #         elif comparator == 'not like':
+    #             where.append(col.notlike(f'%{ftv[0]}%'))
+    #
+    #         elif comparator == 'in':
+    #             where.append(col.in_(ftv))
+    #
+    #
+    #     return where
+
+    def handler_filts(self, *filters):
+        """
+        :param filters: (filts:list,relation:str)
+        :param g_f:
+        :param relation:
+        :return:
+        """
+
+        user_filters = []
+        for filter in filters:
+            filts = filter[0]
+            relation = filter[1]
+            user_filter = []
+            for item in filts:
+
+                where = user_filter
+
+                col = sa.Column(item['columnName'])
+                if item.get('data_type') == 'datetime':
+                    col = func.addHours(col, self.zone_time)
+
+                comparator = item['comparator']
+                ftv = item['ftv']
+                if comparator == '==':
+                    if len(ftv) > 1:
+                        where.append(or_(*[col == v for v in ftv]))
+                    else:
+                        where.append(col == ftv[0])
+                elif comparator == '>=':
+                    where.append(col >= ftv[0])
+                elif comparator == '<=':
+                    where.append(col <= ftv[0])
+                elif comparator == '>':
+                    where.append(col > ftv[0])
+                elif comparator == '<':
+                    where.append(col < ftv[0])
+
+                elif comparator == 'is not null':
+                    where.append(col.isnot(None))
+                elif comparator == 'is null':
+                    where.append(col.is_(None))
+
+                elif comparator == 'like':
+                    where.append(col.like(f'%{ftv[0]}%'))
+
+                elif comparator == 'not like':
+                    where.append(col.notlike(f'%{ftv[0]}%'))
+
+                elif comparator == 'in':
+                    where.append(col.in_(ftv))
+
+                elif comparator == '!=':
+                    where.append(col != ftv[0])
+            if relation == 'and':
+                if user_filter:
+                    user_filters.append(and_(*user_filter))
+            else:
+                if user_filter:
+                    user_filters.append(or_(*user_filter))
+
+        return user_filters
+
+    def property_model(self):
+        event = self.events
+        selectd = getattr(self.user_tbl.c, event['quota'])
+        qry = sa.select(selectd)
+
+        account_id_col = getattr(self.user_tbl.c, '#account_id')
+        binduid_col = getattr(self.user_tbl.c, '#account_id')
+        # 聚合方式
+        analysis = event['analysis']
+
+        if analysis == 'trig_user_num':
+            selectd = [func.count().label('values')]
+        elif analysis == 'distinct_count':
+            selectd = [
+                func.count(sa.distinct(getattr(self.user_tbl.c, event['quota']))).label('values')]
+
+        else:
+            selectd = [
+                func.round(getattr(func, analysis)(getattr(self.user_tbl.c, event['quota'])), 2).label(
+                    'values')]
+
+        where = self.handler_filts((event['filts'], event.get('relation')),
+                                   (self.global_filters, self.global_relation),
+                                   self.ext_filters
+                                   )
+        qry = sa.select((*self.groupby, *selectd)).where(*where)
+
+        qry = qry.group_by(*self.groupby)
+        qry = qry.order_by(sa.Column('values').desc())
+        qry = qry.limit(1000)
+        sql = str(qry.compile(compile_kwargs={"literal_binds": True}))
+        print(sql)
+        result = {'sql': sql,
+                  'groupby': [i.key for i in self.groupby],
+                  'quota': event['quota']
+                  }
+
+        return result

models/user_label.py

@@ -0,0 +1,234 @@
+"""
+本质查出符合条件的用户id
+得到sql 查uid
+"""
+
+import re
+from typing import Tuple
+
+import arrow
+import sqlalchemy as sa
+import json
+
+from fastapi import Depends
+
+import pandas as pd
+
+from sqlalchemy import func, or_, and_, not_
+
+import crud
+import schemas
+from core.config import settings
+from db import get_database
+
+from db.redisdb import get_redis_pool, RedisDrive
+
+
+class UserClusterDef:
+    def __init__(self, game: str, cluster_name: str, data_where: list = None, rdb: RedisDrive = get_redis_pool(),
+                 **kwargs):
+        self.game = game
+        self.rdb = rdb
+        self.cluster_name = cluster_name
+        self.event_tbl = None
+        self.data_where = data_where or []
+        self.kwargs = kwargs
+
+    async def _init_tal(self):
+        res_json = await self.rdb.get(f'{self.game}_event')
+        columns = json.loads(res_json).keys()
+        metadata = sa.MetaData(schema=self.game)
+        self.event_tbl = sa.Table('event', metadata, *[sa.Column(column) for column in columns])
+
+        res_json = await self.rdb.get(f'{self.game}_user')
+        columns = json.loads(res_json).keys()
+        metadata = sa.MetaData(schema=self.game)
+        self.user_tbl = sa.Table('user_view', metadata, *[sa.Column(column) for column in columns])
+
+        self.u_account_id_col = getattr(self.user_tbl.c, '#account_id')
+        self.e_account_id_col = getattr(self.event_tbl.c, '#account_id')
+        self.account_id_col = sa.Column('#account_id')
+
+    async def init(self):
+
+        self.data_in = (
+            await crud.user_label.find_one(get_database(), {'cluster_name': self.cluster_name, 'game': self.game},
+                                           {'qp': 1})).get('qp')
+        await self._init_tal()
+        self.events = self.data_in['user_cluster_def']['events']
+        self.event_relation = self.data_in['user_cluster_def']['event_relation']
+
+    async def handler_filts(self, *filters):
+        """
+
+        :param filters: (filts:list,relation:str)
+        :param g_f:
+        :param relation:
+        :return:
+        """
+
+        user_filters = []
+        event_filters = []
+        for filter in filters:
+            filts = filter[0]
+            relation = filter[1]
+            user_filter = []
+            event_filter = []
+            for item in filts:
+                comparator = item['comparator']
+                if item['tableType'] == 'user':
+                    where = user_filter
+                elif item['tableType'] == 'event':
+                    where = event_filter
+                else:
+                    continue
+
+                tbl = getattr(self, f'{item["tableType"]}_tbl')
+                col = getattr(tbl.c, item['columnName'])
+
+                ftv = item['ftv']
+                if comparator == '==':
+                    if len(ftv) > 1:
+                        where.append(or_(*[col == v for v in ftv]))
+                    else:
+                        where.append(col == ftv[0])
+                elif comparator == '>=':
+                    where.append(col >= ftv[0])
+                elif comparator == '<=':
+                    where.append(col <= ftv[0])
+                elif comparator == '>':
+                    where.append(col > ftv[0])
+                elif comparator == '<':
+                    where.append(col < ftv[0])
+
+                elif comparator == 'is not null':
+                    where.append(col.isnot(None))
+                elif comparator == 'is null':
+                    where.append(col.is_(None))
+
+                elif comparator == 'like':
+                    where.append(col.like(f'%{ftv[0]}%'))
+
+                elif comparator == 'not like':
+                    where.append(col.notlike(f'%{ftv[0]}%'))
+
+                elif comparator == 'in':
+                    where.append(col.in_(ftv))
+
+                elif comparator == '!=':
+                    where.append(col != ftv[0])
+            if relation == 'and':
+                if event_filter:
+                    event_filters.append(and_(*event_filter))
+                if user_filter:
+                    user_filters.append(and_(*user_filter)),
+            else:
+                if event_filter:
+                    event_filters.append(or_(*event_filter))
+                if user_filter:
+                    user_filters.append(or_(*user_filter))
+
+        return event_filters, user_filters
+
+    def to_sql_qry(self):
+        qry = None
+        for event in self.events:
+            event_name = event['event_name']
+            event_name_col = getattr(self.event_tbl.c, '#event_name')
+            analysis = event['prop_quota']['analysis']
+            quota = event['prop_quota']['quota']
+            num = event['num'].split(',')
+            date_type = event.get('date_type', 'dynamic')
+            e_days = event.get('e_days')
+            s_days = event.get('s_days')
+            is_touch = event.get('is_touch', True)
+
+            filts = event['filts']
+            zone = event.get('zone', 8)
+
+            # 账号数据过滤
+            data_where = []
+            filters = []
+            filters.extend(self.data_where)
+            for item in filters:
+                tmp = settings.CK_CALC_SYMBO[item['comparator']](sa.Column(item['columnName']), item['ftv'])
+                data_where.append(tmp)
+
+            event_time_col = func.addHours(getattr(self.event_tbl.c, '#event_time'), zone)
+            date_where = []
+            if date_type == 'static':
+                start_time = event['start_time']
+                end_time = event['end_time']
+                date_where.extend(
+                    [settings.CK_CALC_SYMBO['>='](event_time_col, start_time),
+                     settings.CK_CALC_SYMBO['<='](event_time_col, end_time)]
+                )
+            elif date_type == 'dynamic':
+                start_time = arrow.get().shift(days=-int(s_days)).strftime('%Y-%m-%d 00:00:00')
+                end_time = arrow.get().shift(days=-int(e_days)).strftime('%Y-%m-%d 23:59:59')
+                date_where.extend(
+                    [settings.CK_CALC_SYMBO['>='](event_time_col, start_time),
+                     settings.CK_CALC_SYMBO['<='](event_time_col, end_time)]
+                )
+            else:
+                # 所有时间
+                pass
+
+            uce_calcu_symbol = event['uce_calcu_symbol']
+
+            event_name_where = []
+            if event_name != '*':
+                # 任意事件
+                event_name_where.append(settings.CK_CALC_SYMBO['=='](event_name_col, event_name))
+            if quota != '*':
+                selectd = [self.account_id_col,
+                           func.round(getattr(func, analysis)(getattr(self.event_tbl.c, quota)), 2).label(
+                               'values')
+                           ]
+                qry_tmp = sa.select(self.account_id_col).select_from(
+                    sa.select(selectd).where(*date_where, *event_name_where, *data_where).group_by(
+                        self.e_account_id_col).having(
+                        settings.CK_CALC_SYMBO[uce_calcu_symbol](sa.Column('values'), *num)))
+            else:
+                selectd = [self.account_id_col]
+                qry_tmp = sa.select(self.account_id_col).select_from(
+                    sa.select(selectd).where(*date_where, *event_name_where, *data_where))
+
+            if qry is None:
+                qry = qry_tmp
+            else:
+                if self.event_relation == 'and':
+                    qry = sa.select(self.account_id_col).select_from(
+                        sa.join(qry, qry_tmp, getattr(qry.c, '#account_id') == getattr(qry_tmp.c, '#account_id')))
+                elif self.event_relation == 'or':
+                    qry = sa.select(sa.distinct(self.account_id_col)).select_from(sa.union_all(qry, qry_tmp))
+        # 处理没做过
+        if not is_touch:
+            qry = sa.select(self.u_account_id_col).where(self.u_account_id_col.notin_(qry))
+
+        return qry
+
+    def to_sql(self):
+        qry = self.to_sql_qry()
+        sql = str(qry.compile(compile_kwargs={"literal_binds": True}))
+        print(sql)
+        return sql
+
+    def cluster_user_list(self):
+        sub_qry = self.to_sql_qry()
+        page = self.kwargs.get('page') or 1
+        page -= 1
+        limit = self.kwargs.get('limit', 50)
+        qry = sa.select('*').where(self.u_account_id_col.in_(sub_qry)).order_by(sa.Column('#reg_time')) \
+            .offset(page * limit) \
+            .limit(limit)
+        sql = str(qry.compile(compile_kwargs={"literal_binds": True}))
+        print(sql)
+        return sql
+
+    def cluster_user_count(self):
+        sub_qry = self.to_sql_qry()
+        qry = sa.select(func.count(self.account_id_col).label('values')).select_from(sub_qry)
+        sql = str(qry.compile(compile_kwargs={"literal_binds": True}))
+        print(sql)
+        return sql

models/x_analysis.py

@@ -0,0 +1,203 @@
+from typing import Tuple
+
+import arrow
+import sqlalchemy as sa
+import json
+
+from fastapi import Depends
+
+import pandas as pd
+
+from sqlalchemy import func, or_, and_, not_, MetaData
+
+import crud
+import schemas
+from core.config import settings
+from db import get_database
+from db.redisdb import get_redis_pool, RedisDrive
+
+
+class XAnalysis:
+    def __init__(self, data_in: schemas.CkQuery, game: str):
+        self.data_in = data_in
+        self.game = game
+        self.event_view = dict()
+        self.events = []
+
+        self.global_filters = []
+        self.account_filters = []
+        self.global_relation = 'and'
+        self.date_range = []
+
+        self.ext_filters = (self.data_in.ext_filter.get('filts', []), self.data_in.ext_filter.get('relation', 'and'))
+
+    def _get_global_filters(self):
+        return self.event_view.get('filts') or []   #获取event_view字典里面filts的值，或返回空列表
+
+    async def init(self, *args, **kwargs):
+        if self.data_in.report_id:
+            db = get_database()
+            report = await crud.report.get(db, id=self.data_in.report_id)
+            self.event_view = report['query']['eventView']
+            self.events = report['query']['events']
+            try:
+                e_days = self.event_view['e_days']
+                s_days = self.event_view['s_days']
+            except:
+                # 兼容以前的
+                e_days, s_days = self.event_view['recentDay'].split('-')
+            # self.event_view['endTime'] = arrow.get().shift(days=-int(e_days)+1).strftime('%Y-%m-%d 23:59:59')
+            # self.event_view['startTime'] = arrow.get().shift(days=-int(s_days)+1).strftime('%Y-%m-%d 00:00:00')
+            self.event_view['endTime'] = arrow.get().shift(days=-int(e_days)).strftime('%Y-%m-%d 23:59:59')
+            self.event_view['startTime'] = arrow.get().shift(days=-int(s_days)).strftime('%Y-%m-%d 00:00:00')
+
+        else:
+            self.event_view = self.data_in.eventView
+            self.events = self.data_in.events
+        for d in pd.date_range(self.event_view['startTime'], self.event_view['endTime'], freq='D', tz='UTC'):
+            self.date_range.append(d.date())
+
+        self.global_filters = self._get_global_filters()
+        self.global_relation = self.event_view.get('relation', 'and')
+
+        # 用户自带过滤
+        if 'data_where' in kwargs:
+            self.account_filters = kwargs['data_where'].get(self.game, [])
+
+    def handler_filts(self, *filters):
+        """
+        :param filters: (filts:list,relation:str)
+        :param g_f:
+        :param relation:
+        :return:
+        """
+
+        event_filters = []
+        for filter in filters:
+            filts = filter[0]
+            relation = filter[1]
+            event_filter = []
+            for item in filts:
+
+                where = event_filter
+
+                col = sa.Column(item['columnName'])
+
+                comparator = item['comparator']
+                ftv = item['ftv']
+                if comparator == '==':
+                    if len(ftv) > 1:
+                        where.append(or_(*[col == v for v in ftv]))
+                    else:
+                        where.append(col == ftv[0])
+                elif comparator == '>=':
+                    where.append(col >= ftv[0])
+                elif comparator == '<=':
+                    where.append(col <= ftv[0])
+                elif comparator == '>':
+                    where.append(col > ftv[0])
+                elif comparator == '<':
+                    where.append(col < ftv[0])
+
+                elif comparator == 'is not null':
+                    where.append(col.isnot(None))
+                elif comparator == 'is null':
+                    where.append(col.is_(None))
+
+                elif comparator == 'like':
+                    where.append(col.like(f'%{ftv[0]}%'))
+
+                elif comparator == 'not like':
+                    where.append(col.notlike(f'%{ftv[0]}%'))
+
+                elif comparator == 'in':
+                    where.append(col.in_(ftv))
+
+                elif comparator == '!=':
+                    where.append(col != ftv[0])
+            if relation == 'and':
+                if event_filter:
+                    event_filters.append(and_(*event_filter))
+            else:
+                if event_filter:
+                    event_filters.append(or_(*event_filter))
+
+        return event_filters
+
+
+    def ltv_model_sql(self):
+        days = (arrow.get(self.event_view['endTime']).date() - arrow.get(self.event_view['startTime']).date()).days
+        quota = self.event_view['quota']
+        select_ltv = []
+        sumpay = []
+        sum_money = []
+        # for i in range(1, days + 2):
+        ltv_n = [*[k for k in range(1, 61)], 70, 75, 80, 85, 90, 95, 100, 110, 120, 150, 180, 210, 240, 270, 300, 360]
+        for i in ltv_n:
+            # select_ltv.append(func.round(sa.Column(f'sumpay_{i}') / sa.Column('cnt1'), 2).label(f'LTV{i}'))
+            select_ltv.append(
+                f"if(dateDiff('day', reg.date, now())<{i - 1}, '-',toString(round(sumpay_{i} / cnt1, 2))) AS LTV{i}")
+            sumpay.append(f"sum(if(dateDiff('day', a.date, b.date) < {i}, money, 0)) as sumpay_{i}")
+            sum_money.append(f"sumpay_{i}")
+        # qry = sa.select(*select_ltv)
+        # select_ltv_str = str(qry.compile(compile_kwargs={"literal_binds": True}))
+        # select_ltv_str = select_ltv_str.split('SELECT ')[1]
+        sumpay_str = ','.join(sumpay)
+        select_ltv_str = ','.join(select_ltv)
+        sum_money_str = ','.join(sum_money)
+
+        where = [
+            sa.Column('date') >= self.event_view['startTime'].split(' ')[0],
+            sa.Column('date') <= self.event_view['endTime'].split(' ')[0]
+        ]
+        if quota == '#distinct_id':
+            where.append(sa.Column('is_new_device') == 1)
+
+        qry = sa.select().where(*where)
+        sql = str(qry.compile(compile_kwargs={"literal_binds": True}))
+        where_str = sql.split('WHERE ')[1]
+
+        where_order = self.handler_filts((self.global_filters, self.global_relation))   #global_relation就是  and
+        where_order_str = 1
+        if where_order:
+            qry = sa.select().where(*where_order)
+            sql = str(qry.compile(compile_kwargs={"literal_binds": True}))
+            where_order_str = sql.split('WHERE ')[1]
+
+        where_account = self.handler_filts((self.account_filters, 'and'), self.ext_filters)
+        where_account_str = 1
+        if where_account:
+
+            qry = sa.select().where(*where_account)
+            sql = str(qry.compile(compile_kwargs={"literal_binds": True}))
+            where_account_str = sql.split('WHERE ')[1]
+        sql = f"""SELECT reg.date as date,
+               cnt1,
+               {select_ltv_str},
+               {sum_money_str}
+        FROM (SELECT toDate(addHours(`#event_time`, `#zone_offset`)) as date, uniqExact(`{quota}`) cnt1
+              FROM {self.game}.event  
+              where `#event_name` = 'create_account'
+                AND {where_str} AND {where_account_str} 
+              GROUP BY toDate(addHours(`#event_time`, `#zone_offset`))) as reg
+                 left join
+             (select a.date,
+                     {sumpay_str}
+              from (SELECT toDate(addHours(`#event_time`, `#zone_offset`)) as date, `{quota}`
+                    FROM {self.game}.event
+                    where `#event_name` = 'create_account'
+                      AND {where_str}  AND {where_account_str} ) as a
+                       left join (select `{quota}`, unitPrice/100 as money, toDate(addHours(`#event_time`, `#zone_offset`)) as date
+                                  from {self.game}.event
+                                  where `#event_name` = 'pay' and {where_order_str}  AND {where_account_str}) b
+                                 on a.`{quota}` = b.`{quota}`
+              group by a.date) log on reg.date = log.date
+        order by date
+        """
+        print(sql)
+        return {'sql': sql, 'quota': quota,
+                'start_date': self.event_view['startTime'][:10],
+                'end_date': self.event_view['endTime'][:10],
+                'date_range': self.date_range,
+                'ltv_n': ltv_n
+                }

schemas/__init__.py

@@ -0,0 +1,27 @@
+from .msg import Msg
+from .user import *
+from .project import *
+from .folder import *
+from .space import *
+from .dashboard import *
+from .report import *
+from .authotity import *
+from .table_struct import *
+from .data_auth import *
+from .data_attr import *
+from .sql import *
+from .api_log import *
+from .event_mana import *
+from .xquery import *
+from .api_list import *
+from .role import *
+from .check_data import *
+from .userlabel import *
+from .select_map import *
+from .project_number import *
+from .proid_map import *
+from .api_board import *
+from .url_list import *
+from .user_url import *
+from .api_module import *
+from .event_list import *

schemas/api_board.py

@@ -0,0 +1,12 @@
+from typing import Any, List, Union
+
+from pydantic import BaseModel, Field
+
+from schemas import DBBase
+from typing import Optional
+
+
+class Api_board(BaseModel):
+    api_path: str = None
+    api_name: str = None
+    name: str

schemas/api_list.py

@@ -0,0 +1,37 @@
+from typing import Any, List, Union
+
+from pydantic import BaseModel, Field
+
+from schemas import DBBase
+from typing import Optional
+
+
+class ApiBase(BaseModel):
+    path: str = None
+    name: str = None
+    desc: str = None
+
+
+class AddApi(ApiBase):
+    path: str
+    name: str
+    desc: str = None
+
+
+class UpdateApi(BaseModel):
+    path: str
+    name: str
+
+
+class AddApiDB(DBBase, AddApi):
+    pass
+
+
+class DelApi(BaseModel):
+    ids: List[str] = Field(..., description='要删除的id')
+
+
+class EditApi(BaseModel):
+    id: str = Field(..., description='要编辑的id')
+    name: str
+    desc: str

schemas/api_log.py

@@ -0,0 +1,9 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class ApiLogInsert(BaseModel):
+    api: str
+    ms: int
+    user_id: str

schemas/api_module.py

@@ -0,0 +1,18 @@
+from typing import Any, List, Union
+
+from pydantic import BaseModel, Field
+
+from schemas import DBBase
+from typing import Optional
+
+
+class Url_module(BaseModel):
+    auth_id: str = None
+    path_name: str = None
+    api_list: List[str] = None
+    api_name: List[str] = None
+    state: List[bool] = None
+
+class Add_module(BaseModel):
+    auth_id: str
+    url:str

schemas/authotity.py

@@ -0,0 +1,82 @@
+from enum import Enum
+from typing import List
+
+from pydantic import BaseModel
+
+
+class AddRoleForUserInDomain(BaseModel):
+    username: str
+    role_id: str
+    game: str
+    auth_id: str
+
+
+class AddRoleForUsersInDomain(BaseModel):
+    data: List[AddRoleForUserInDomain]
+
+
+class GetPermissionsForUserInDomain(BaseModel):
+    role_id: str
+    game: str
+
+
+class DeleteRolesForUserInDomain(BaseModel):
+    username: str
+    role_id: str
+    game: str
+
+
+class Policy(BaseModel):
+    role_id: str
+    game: str
+    path: str
+    act: str = '*'
+
+
+class AddPolicy(BaseModel):
+    path_list: List[str]
+    role_id: str
+    game: str
+    act: str = '*'
+
+
+class DelPolicy(Policy):
+    pass
+
+
+class Ptype(str, Enum):
+    p = 'p'
+    g = 'g'
+
+
+class CasbinRoleCreate(BaseModel):
+    role_name: str
+    role_api: List[str]
+
+
+class CasbinDB(BaseModel):
+    ptype: Ptype
+    v0: str
+    v1: str
+    v2: str
+
+
+class AccountCreate(BaseModel):
+    username: str
+    role_name: str
+    # nickname: str
+    data_auth_id: str
+
+
+class AccountsCreate(BaseModel):
+    accounts: List[AccountCreate]
+    project_id: str
+
+
+class AccountDeleteUser(BaseModel):
+    name: str
+
+
+class AccountSetRole(BaseModel):
+    name: str
+    role_name: str

schemas/base.py

@@ -0,0 +1,28 @@
+import uuid
+from typing import Optional, Union
+
+from bson import ObjectId
+from pydantic import BaseModel, Field, validator
+from utils import *
+
+
+# # mongodb  _id 类型
+# class OId(ObjectId):
+#     @classmethod
+#     def __get_validators__(cls):
+#         yield cls.validate
+#
+#     @classmethod
+#     def validate(cls, v):
+#         try:
+#             return ObjectId(v)
+#         except:
+#             raise ValueError('无效的格式')
+
+
+class DBBase(BaseModel):
+    id: str = Field(None, alias='_id')
+
+    @validator('id', pre=True, always=True)
+    def default_id(cls, v):
+        return v or get_uid()

schemas/check_data.py

@@ -0,0 +1,20 @@
+from pydantic import BaseModel
+
+
+class CheckData(BaseModel):
+    db_name: str
+    event_name: str
+    is_unique: bool
+    props: dict
+    default_field: dict = dict()
+    where: dict = dict()
+    game:str
+
+
+class AddTemplate(BaseModel):
+    check: CheckData
+    title: str
+
+
+class DelTemplate(BaseModel):
+    title: str

schemas/dashboard.py

@@ -0,0 +1,105 @@
+import uuid
+from datetime import datetime
+from enum import Enum
+from typing import List, Dict
+
+from pydantic import BaseModel
+
+from schemas import DBBase
+
+
+class DashboardBase(BaseModel):
+    name: str = None
+
+
+# 解析请求json 创建项目
+class DashboardCreate(DashboardBase):
+    name: str
+    project_id: str
+    # cat: str
+    pid: str
+
+
+class ReadDashboard(BaseModel):
+    id: str
+
+
+class DashboardDelete(BaseModel):
+    ids: List[str]
+
+
+class Report(BaseModel):
+    name: str = None
+    report_id: str = None
+    graph_type: str = None
+    ascending: bool = None
+    model: str = None
+    graph_size: str = None
+    sort: int = None
+    modelswitch: bool = None
+    avesumdata: bool = True
+    daydata: bool = True
+    reverseorder: bool = True
+
+
+class EditShowReport(BaseModel):
+    dashboard_id: str
+    config: Report
+
+
+class Category(str, Enum):
+    project = 'kanban'
+    space = 'space'
+
+
+class EditDashboard(BaseModel):
+    dashboard_id: str
+    new_name: str
+
+
+class DashboardMove(BaseModel):
+    source_ids: List[str]
+    dest_pid: str
+    cat: Category
+
+
+class Sort(BaseModel):
+    dashboard_id: str
+    sort: int
+
+
+class DashboardSort(BaseModel):
+    sort: List[Sort]
+
+
+class DashboardCopy(BaseModel):
+    source_ids: List[str]
+    dest_project_id: str
+
+class DashboardCopyToSpace(BaseModel):
+    source_ids: List[str]
+    project_id: str
+    dest_space_id: str
+
+class AddReport(DBBase):
+    report_ids: List[Report]
+
+
+class DelReport(DBBase):
+    report_id: str
+
+
+class EditReport(DBBase):
+    report: Report
+
+
+# --------------------------------------------------------------
+# 数据库模型
+class DashboardDB(DBBase):
+    name: str
+    user_id: str
+    project_id: str
+    # cat: Category
+    reports: List[str] = []
+    pid: str
+    create_date: datetime = datetime.now()

schemas/data_attr.py

@@ -0,0 +1,14 @@
+from pydantic import BaseModel
+
+
+class DataAttrEdit(BaseModel):
+    name: str
+    show_name: str
+    is_show: bool
+    cat: str
+
+class Add_attr(BaseModel):
+    cat: str
+    new_attribute: str
+    state: str
+    data_type: str

schemas/data_auth.py

@@ -0,0 +1,24 @@
+from typing import List
+
+from pydantic import BaseModel
+
+
+class DataAuthCreate(BaseModel):
+    title: str
+    data: List[str] = []
+
+
+class DataAuthEdit(BaseModel):
+    data_auth_id: str
+    title: str
+    data: List[str] = []
+
+
+class DataAuthSet(BaseModel):
+    username: str
+    data_auth_id: str
+
+
+class LoadProQuotas(BaseModel):
+    event_name: str
+    model: str = None

schemas/event_list.py

@@ -0,0 +1,10 @@
+from typing import List, Dict
+from pydantic import BaseModel
+
+class Event_list(BaseModel):
+    game:str
+    details:List[Dict]
+
+class Details(BaseModel):
+    event:str
+    event_name:str

schemas/event_mana.py

@@ -0,0 +1,8 @@
+from pydantic import BaseModel
+
+
+class EventMateEdit(BaseModel):
+    event_name: str
+    show_name: str
+    is_show: bool
+    desc: str

schemas/folder.py

@@ -0,0 +1,41 @@
+import uuid
+from datetime import datetime
+from enum import Enum
+from typing import List
+
+from pydantic import BaseModel
+
+from schemas import DBBase
+
+
+class FolderBase(BaseModel):
+    name: str = None
+
+
+# 解析请求json 创建项目
+class FolderCreate(FolderBase):
+    name: str
+    project_id: str
+    cat: str
+    pid: str
+
+
+class FolderDelete(DBBase):
+    pass
+
+
+class Category(str, Enum):
+    project = 'kanban'
+    space = 'space'
+
+
+# --------------------------------------------------------------
+# 数据库模型
+class FolderDB(DBBase):
+    name: str
+    user_id: str
+    project_id: str
+    cat: Category
+    pid: str
+    members: List[str] = []
+    create_date: datetime = datetime.now()

schemas/msg.py

@@ -0,0 +1,9 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class Msg(BaseModel):
+    code: int
+    msg: str
+    data: Any

schemas/project.py

@@ -0,0 +1,71 @@
+import uuid
+from datetime import datetime
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
+
+from schemas import DBBase
+
+
+class ProjectBase(BaseModel):
+    name: str = None
+
+
+class MemberRole(BaseModel):
+    username: str
+    user_id: str
+    role_name: str
+    data_auth_id: str
+
+
+class ProjectAddMember(BaseModel):
+    members: List[MemberRole]
+    project_id: str
+
+
+class ProjectMember(BaseModel):
+    members: List[str]
+    project_id: str
+
+
+class ProjectDetail(BaseModel):
+    project_id: str
+
+
+class ProjectClean(BaseModel):
+    project_id: str
+
+class Import_project(BaseModel):
+    game: str
+    games: str
+
+class ProjectRename(BaseModel):
+    project_id: str
+    rename: str
+
+
+class ProjectDelMember(BaseModel):
+    project_id: str
+    role: str
+    username: str
+
+
+# 解析请求json 创建项目
+class ProjectCreate(ProjectBase):
+    name: str = Field(..., title='项目名')
+    game: str = Field(..., title='游戏代号')
+    #qudao:str = Field(...,title='渠道')
+
+# 查询某个项目看板
+class ProjectKanban(DBBase):
+    pass
+
+
+# --------------------------------------------------------------
+# 数据库模型
+class ProjectDB(DBBase):
+    name: str
+    game: str
+    user_id: str
+    members: List[str] = []
+    create_date: datetime = datetime.now()

schemas/project_number.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel
+from typing import List
+
+
+class ProjectnumberList(BaseModel):
+    main_channel: str
+    ditch: str
+
+
+class ProjectnumberInsert(BaseModel):
+    game: str
+    ditch: List[ProjectnumberList]
+    name: str
+
+class AddProjectnumber(BaseModel):
+    game: str
+    ditch: List[ProjectnumberInsert]

schemas/report.py

@@ -0,0 +1,57 @@
+import json
+import uuid
+from datetime import datetime
+from enum import Enum
+from typing import List
+
+from pydantic import BaseModel, validator, Json
+
+from schemas import DBBase
+
+
+class ReportBase(BaseModel):
+    name: str = None
+    query: str = None
+    project_id: str = None
+
+
+class ReportCreate(ReportBase):
+    name: str
+    desc: str
+    project_id: str
+    query: dict
+    cat: str
+
+
+class ReportEdit(BaseModel):
+    report_id: str
+    query: dict
+    name: str
+    desc: str
+
+
+class ReportCopy(BaseModel):
+    report_ids: List[str]
+    dest_project_id: str
+
+
+class ReportDelete(DBBase):
+    pass
+
+
+class ReportRead(BaseModel):
+    project_id: str
+    report_id: List = []
+    dashboard_id: str = None
+
+
+# --------------------------------------------------------------
+# 数据库模型
+class ReportDB(DBBase):
+    name: str
+    user_id: str
+    project_id: str
+    desc: str
+    query: dict
+    cat: str
+    create_date: datetime = datetime.now()

schemas/role.py

@@ -0,0 +1,37 @@
+from typing import List
+
+from pydantic import Field
+from pydantic.main import BaseModel
+
+from schemas import DBBase
+
+
+class RoleBase(BaseModel):
+    game: str = None
+    name: str = None
+    desc: str = None
+
+
+class AddRole(BaseModel):
+    game: str
+    name: str
+    desc: str
+
+
+class AddRoleDB(DBBase, AddRole):
+    pass
+
+
+class DelRole(BaseModel):
+    ids: List[str] = Field(..., description='要删除的id')
+
+
+class EditRole(BaseModel):
+    role_id: str = Field(..., description='要编辑的id')
+    name: str = None
+    desc: str = None
+
+
+class OwnerList(BaseModel):
+    owners: list
+    account_name: str

schemas/select_map.py

@@ -0,0 +1,13 @@
+from typing import Any, List, Union, Dict
+
+from pydantic import BaseModel, Field
+
+
+class SelectMap(BaseModel):
+    game: str
+    attr_name: str
+    map_: List[Dict]
+
+
+class SelectAttr(BaseModel):
+    attr_name: str

schemas/space.py

@@ -0,0 +1,59 @@
+import uuid
+from datetime import datetime
+from enum import Enum
+from typing import List
+
+from pydantic import BaseModel
+
+from schemas import DBBase
+
+
+class SpaceBase(BaseModel):
+    name: str = None
+
+
+class Authority(str, Enum):
+    rw = 'rw'
+    r = 'r'
+
+
+class Member(BaseModel):
+    user_id: str
+    authority: Authority
+
+
+# 解析请求json 创建项目
+class SpaceCreate(SpaceBase):
+    name: str
+    project_id: str
+    members: List[Member] = []
+    is_all_member: bool = False
+    authority: Authority = 'r'
+
+
+class SpaceDelete(DBBase):
+    pass
+
+
+class SpaceDetail(BaseModel):
+    space_id: str
+
+
+class SpaceRename(BaseModel):
+    space_id: str
+    new_name: str
+
+
+class AddSpaceMembers(BaseModel):
+    space_id: str
+    members: List[Member]
+
+
+# --------------------------------------------------------------
+# 数据库模型
+class SpaceDB(DBBase):
+    name: str
+    user_id: str
+    project_id: str
+    members: List[Member] = []
+    create_date: datetime = datetime.now()

schemas/sql.py

@@ -0,0 +1,37 @@
+from typing import List, Union, Dict
+
+from pydantic import BaseModel
+from typing import Optional
+
+
+class Sql(BaseModel):
+    sql: str
+
+
+class CkQuery(BaseModel):
+    eventView: dict = None
+    events: Union[List[dict], dict] = None
+    report_id: str = None
+    ext_filter: dict = dict()
+
+class Ck_seek_user(BaseModel):
+    user_arrt_title: str  # 用户属性
+    user_arrt_id: str  # 用户属性id
+    user_arrt_type: str  # 用户属性type
+    comparator_title: str  # 筛选条件
+    comparator_id: str  # 筛选条件id
+    condition: str  # 手动输入条件，区间用~符号隔开，如0~10
+    start_time: str  # 开始时间
+    end_time: str  # 结束时间
+    pages: int = 1   # 分页的当前页
+
+
+class Ck_solo_user(BaseModel):
+    account_id : str # #account_id
+    start_time: str  # 开始时间 例：2022-04-02
+    end_time: str  # 结束时间
+    event_list: List[Dict] =None#事件名
+
+class Times(BaseModel):
+    start_time: str  # 开始时间 例：2022-04-02 00:00:00
+    end_time: str  # 结束时间

schemas/table_struct.py

@@ -0,0 +1,13 @@
+from enum import Enum
+
+from pydantic import BaseModel
+
+
+class TableEnum(str, Enum):
+    event = 'event'
+    user = 'user'
+
+
+class GetTable(BaseModel):
+
+    name: TableEnum

schemas/token.py

@@ -0,0 +1,14 @@
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class Token(BaseModel):
+    token: str
+    code: int
+    name: str
+    email: str
+    msg: str
+
+
+

schemas/url_list.py

@@ -0,0 +1,61 @@
+from typing import Any, List, Union
+
+from pydantic import BaseModel, Field
+
+from schemas import DBBase
+from typing import Optional
+
+
+class Url_list(BaseModel):
+    name: str = None
+    auth_id: str = None
+    path_name: str = None
+    api_list: List[str] = None
+    api_name: List[str] = None
+    state: List[bool] = None
+    system: int = None
+
+
+class Url_lists(BaseModel):
+    name: str = None
+    auth_id: str = None
+    path_name: str = None
+    api_list: List[str] = None
+    api_name: List[str] = None
+    state: List[bool] = None
+    system: int = None
+    game: str = None
+
+
+class Url_data(BaseModel):
+    api_list: List[str] = None
+    api_name: List[str] = None
+    path_name: str = None
+    stath_name: List[bool] = None
+
+
+class Datalist(BaseModel):
+    path: str
+    path_name: str
+    role_id: str
+    system: int
+
+
+class Add_role(BaseModel):
+    path_name: List[str]
+    system: int
+    name: str
+
+class Del_role(BaseModel):
+    path: str
+    path_name: str
+    role_id: str
+class Editname(BaseModel):
+    role_id: str #= Field(..., description='要编辑的id')
+    name: str = None
+    desc: str = None
+
+class Del_roles(BaseModel):
+    game:str
+    role_id:str
+    username:str