From 08e87a4f4aeb3a91aed0694bc7b7ce22c7a3f2cb Mon Sep 17 00:00:00 2001 From: wuaho Date: Thu, 27 May 2021 18:10:28 +0800 Subject: [PATCH] 1 --- api/api_v1/endpoints/authority.py | 2 +- api/api_v1/endpoints/data_auth.py | 22 +++++++++----- api/api_v1/endpoints/project.py | 19 ++++++++---- api/api_v1/endpoints/query.py | 8 ++--- crud/crud_authority.py | 2 +- crud/crud_data_attr.py | 3 +- db/init_db.py | 15 ++++++---- models/to_sql.py | 49 +++++++++++++++++++++++++++---- rbac_model.conf | 2 +- schemas/project.py | 1 + 10 files changed, 88 insertions(+), 35 deletions(-) diff --git a/api/api_v1/endpoints/authority.py b/api/api_v1/endpoints/authority.py index 7b57369..3990d7a 100644 --- a/api/api_v1/endpoints/authority.py +++ b/api/api_v1/endpoints/authority.py @@ -232,7 +232,7 @@ async def all_role(request: Request, roles = await crud.authority.find_many(db, role_name={'$exists': 1}, game='*') sys_data = [{'role': item['v1'], 'title': item['role_name'], 'id': str(item['_id'])} for item in roles] for item in sys_data: - q = await crud.authority.get_role_dom_authority(db, item['role'], dom='*', api_data=api_data) + q = await crud.authority.get_role_dom_authority(db, item['role'], dom=game, api_data=api_data) item['authority'] = [{'title': k, 'child': v} for k, v in q.items()] data = { diff --git a/api/api_v1/endpoints/data_auth.py b/api/api_v1/endpoints/data_auth.py index 9f7b9d4..2aaeb28 100644 --- a/api/api_v1/endpoints/data_auth.py +++ b/api/api_v1/endpoints/data_auth.py @@ -87,7 +87,7 @@ async def my_event(request: Request, """获取自己的事件权限""" # data_auth_id = await crud.authority.get_data_auth_id(db, game, request.user.username) - data_auth_id = await crud.authority.get_data_auth_id(db, game, 'sddfdaa') + data_auth_id = await crud.authority.get_data_auth_id(db, game, 'wuhao') my_data_auth = await crud.data_auth.get(db, ObjectId(data_auth_id)) all_filed = await rdb.get(f'{game}_event') @@ -109,13 +109,10 @@ async def my_event(request: Request, 'category': settings.CK_FILTER.get(settings.CK_TYPE_DICT.get(all_filed.get(item))) or [] } for item in all_filed] - # filter_by = [{ - # 'id': item, - # 'data_type': settings.CK_TYPE_DICT.get(all_filed.get(item)), - # 'title': data_attr.get(item, {}).get('show_name') or item, - # 'category': settings.CK_FILTER.get(settings.CK_TYPE_DICT.get(all_filed.get(item))) or [] - # } for item in all_filed] - + deserialization = { + 'event_attr': {}, + 'event_filter': {} + } for k, v in event_dict.items(): event_attr = [{ 'id': '*', @@ -152,12 +149,17 @@ async def my_event(request: Request, 'category': settings.CK_OPERATOR.get(data_type) or [] } ) + event_filter.append({ 'id': item, 'data_type': data_type, 'title': title, 'category': settings.CK_FILTER.get(data_type) or [] }) + + deserialization['event_attr'][k] = [{'id': 'event', 'title': '事件属性', 'category': event_attr}] + deserialization['event_filter'][k] = [{'id': 'event', 'title': '事件属性', 'category': event_filter}] + event.append({ 'event_name': k, 'event_attr': [{'id': 'event', 'title': '事件属性', 'category': event_attr}], @@ -166,6 +168,10 @@ async def my_event(request: Request, ) res = { + 'operator': settings.CK_OPERATOR, + 'filter': settings.CK_FILTER, + 'deserialization': deserialization, + 'analysis': [{'id': 'event', 'title': '默认分组', 'category': event diff --git a/api/api_v1/endpoints/project.py b/api/api_v1/endpoints/project.py index e1d9758..049af7d 100644 --- a/api/api_v1/endpoints/project.py +++ b/api/api_v1/endpoints/project.py @@ -42,9 +42,9 @@ async def create( ) await crud.folder.create(db, folder, user_id=current_user.id) - # 创建全部数据权限 - data_auth = schemas.DataAuthCreate(name='全部', data=['*']) - await crud.data_auth.create(db, data_auth, data_in.game) + # # 创建全部数据权限 + # data_auth = schemas.DataAuthCreate(name='全部', data=['*']) + # await crud.data_auth.create(db, data_auth, data_in.game) # 新建项目管理员权限 role_name = f'{data_in.game}_admin' @@ -52,7 +52,7 @@ async def create( casbin_enforcer.add_policy(role_name, role_dom, '*', '*') await crud.authority.create(db, 'p', role_name, role_dom, '*', '*') # 添加角色 - await crud.authority.create(db, 'g', settings.SUPERUSER_NAME, role_name, '*', '*', role_name='项目管理员', game=role_dom) + await crud.authority.create(db, 'g', settings.SUPERUSER_NAME, role_name, '*', '*', role_name='系统项目管理员', game='*') return schemas.Msg(code=0, msg='创建成功') @@ -108,7 +108,6 @@ async def rename_project(request: Request, return schemas.Msg(code=0, msg='ok', data=res) -# todo 要修改 @router.post("/add_members") async def add_members(request: Request, game: str, @@ -125,6 +124,14 @@ async def add_members(request: Request, schemas.DataAuthSet(username=item.username, data_auth_id=item.data_auth_id), game) + folder = schemas.FolderCreate( + name='未分组', + project_id=data_in.project_id, + cat='kanban', + pid=data_in.project_id, + ) + await crud.folder.create(db, folder, user_id=item.user_id) + await crud.project.add_members(db, schemas.ProjectMember(project_id=data_in.project_id, members=[item.username for item in data_in.members])) @@ -187,7 +194,7 @@ async def members(request: Request, current_user: schemas.UserDB = Depends(deps.get_current_user) ): """删除项目成员""" - casbin_enforcer.delete_roles_for_user_in_domain(data_in.username,data_in.role,game) + casbin_enforcer.delete_roles_for_user_in_domain(data_in.username, data_in.role, game) await crud.project.del_members(db, data_in) await crud.authority.delete(db, ptype='g', v2=game, v0=data_in.username) return schemas.Msg(code=0, msg='ok') diff --git a/api/api_v1/endpoints/query.py b/api/api_v1/endpoints/query.py index db3a9c0..2edb391 100644 --- a/api/api_v1/endpoints/query.py +++ b/api/api_v1/endpoints/query.py @@ -68,12 +68,8 @@ async def event_model( date_range = item['date_range'] q['date_range'] = date_range df = await ckdb.query_dataframe(sql) - if df.shape[0]==0: + if df.shape[0] == 0: return schemas.Msg(code=0, msg='ok', data=q) - df['date'] = df['date'].apply(lambda x: str(x)) - # todo 时间粒度 暂时按天 - df['date'] = df['date'].apply( - lambda x: pd.Timestamp(year=int(x[:4]), month=int(x[4:6]), day=int(x[6:])).strftime('%Y-%m-%d')) if groupby: # 有分组 @@ -98,6 +94,6 @@ async def event_model( concat_data.append((i, 0)) df = pd.concat([df, pd.DataFrame(concat_data, columns=df.columns)]) q['values'].append(df['values'].to_list()) - + q['date_range'] = [d.strftime('%Y-%m/%d %H:%M:%S') for d in q['date_range']] res.append(q) return schemas.Msg(code=0, msg='ok', data=res) diff --git a/crud/crud_authority.py b/crud/crud_authority.py index 641d0a5..2d1c51e 100644 --- a/crud/crud_authority.py +++ b/crud/crud_authority.py @@ -41,7 +41,7 @@ class CRUDAuthority(CRUDBase): async def get_role_dom_authority(self, db, role, dom, api_data): selected_api = {item['v2'] for item in await self.find_many(db, v0=role, v1=dom)} - anonymous_api = {item['v2'] for item in await self.find_many(db, v0='anonymous')} + anonymous_api = {item['v2'] for item in await self.find_many(db, v0='*')} api_data = deepcopy(api_data) diff --git a/crud/crud_data_attr.py b/crud/crud_data_attr.py index b8df688..8618db7 100644 --- a/crud/crud_data_attr.py +++ b/crud/crud_data_attr.py @@ -12,7 +12,8 @@ __all__ = 'data_attr', class CRUDDataAttr(CRUDBase): async def edit_data_attr(self, db: AsyncIOMotorDatabase, game: str, data_id: schemas.DataAttrEdit): - await self.update_one(db, {'game': game, 'cat': data_id.cat}, {'$set': data_id.dict()}, upsert=True) + await self.update_one(db, {'game': game, 'cat': data_id.cat, 'name': data_id.name}, {'$set': data_id.dict()}, + upsert=True) async def create_index(self, db: AsyncIOMotorDatabase): await db[self.coll_name].create_index( diff --git a/db/init_db.py b/db/init_db.py index 35855dc..dcd66bf 100644 --- a/db/init_db.py +++ b/db/init_db.py @@ -44,12 +44,17 @@ async def report_index(): await crud.report.create_index(db) +async def data_attr_index(): + await crud.data_attr.create_index(db) + + async def authority_init(): await crud.authority.create_index(db) - await crud.authority.create(db, 'p', 'anonymous', '*', '/docs', '*') - await crud.authority.create(db, 'p', 'anonymous', '*', '/openapi.json', '*') - await crud.authority.create(db, 'p', 'anonymous', '*', '/api/v1/user/login', '*') - await crud.authority.create(db, 'p', 'anonymous', '*', '/docs', '*') + await crud.authority.create(db, 'p', '*', '*', '/docs', '*') + await crud.authority.create(db, 'p', '*', '*', '/openapi.json', '*') + await crud.authority.create(db, 'p', '*', '*', '/api/v1/user/login', '*') + await crud.authority.create(db, 'p', '*', '*', '/docs', '*') + await crud.authority.create(db, 'p', '*', '*', '/api/v1/project/', '*') async def main(): @@ -60,7 +65,7 @@ async def main(): await dashboard_index() await report_index() await authority_init() - + await data_attr_index() loop = asyncio.get_event_loop() loop.run_until_complete(main()) diff --git a/models/to_sql.py b/models/to_sql.py index cfd0dee..bb41778 100644 --- a/models/to_sql.py +++ b/models/to_sql.py @@ -1,9 +1,40 @@ +import datetime from typing import List, Tuple import sqlalchemy as sa from sqlalchemy.sql import func from sqlalchemy import create_engine, column, and_, desc, table, or_ import pandas as pd +TIME_ZONE_MAP = { + 8: 'Asia/Shanghai' +} + +PROPHET_TIME_GRAIN_MAP = { + "PT1S": "S", + "PT1M": "min", + "PT5M": "5min", + "PT10M": "10min", + "PT15M": "15min", + "PT0.5H": "30min", + "PT1H": "H", + "P1D": "D", + "P1W": "W", + "P1M": "M", +} + +TIME_GRAIN_EXPRESSIONS = { + 'PT1S': lambda col, zone: func.toStartOfSecond(func.toTimeZone(col, zone)).label('date'), + 'PT1M': lambda col, zone: func.toStartOfMinute(func.toTimeZone(col, zone)).label('date'), + 'PT5M': lambda col, zone: func.toStartOfFiveMinute(func.toTimeZone(col, zone)).label('date'), + 'PT10M': lambda col, zone: func.toStartOfTenMinutes(func.toTimeZone(col, zone)).label('date'), + 'PT15M': lambda col, zone: func.toStartOfFifteenMinutes(func.toTimeZone(col, zone)).label('date'), + # 'PT0.5H': lambda col, zone: func.toStartOfMinute(func.toTimeZone(col, zone)).label('date'), + 'PT1H': lambda col, zone: func.toStartOfHour(func.toTimeZone(col, zone)).label('date'), + 'P1D': lambda col, zone: func.toStartOfDay(func.toTimeZone(col, zone)).label('date'), + 'P1W': lambda col, zone: func.toMonday(func.toTimeZone(col, zone)).label('date'), + 'P1M': lambda col, zone: func.toStartOfMonth(func.toTimeZone(col, zone)).label('date'), +} + class ToSql: def __init__(self, data: dict, db_name: str, table_name: str, columns: List[str]): @@ -18,9 +49,13 @@ class ToSql: def gen_columns(self, columns): return {col: column(col) for col in columns} + def get_zone_time(self): + return int(self.event_view.get('zone_time')) + def get_date_range(self) -> Tuple[str, str]: start_data: str = self.event_view.get('startTime') end_data: str = self.event_view.get('endTime') + return start_data, end_data def get_global_filters(self): @@ -31,7 +66,7 @@ class ToSql: return [item['column_id'] for item in self.event_view.get('groupBy')] def get_time_particle_size(self): - return self.event_view.get('timeParticleSize') or 'day' + return self.event_view.get('timeParticleSize') or 'P1D' def get_sql_query_event_model(self): """只是查event表""" @@ -40,10 +75,10 @@ class ToSql: select_exprs = [self.columns.get(item) for item in select_exprs] time_particle_size = self.get_time_particle_size() start_data, end_data = self.get_date_range() - date_range = [] - if time_particle_size == 'day': - select_exprs.insert(0, func.toYYYYMMDD(self.columns['#event_time']).label('date')) - date_range = pd.date_range(start_data, end_data, freq='D').strftime('%Y-%m-%d').tolist() + time_zone = TIME_ZONE_MAP[self.get_zone_time()] + select_exprs.insert(0, TIME_GRAIN_EXPRESSIONS[time_particle_size](self.columns['#event_time'], time_zone)) + date_range = pd.date_range(start_data, end_data, freq=PROPHET_TIME_GRAIN_MAP[time_particle_size], + tz=time_zone).tolist() groupby = [item.name for item in select_exprs] for event in self.events: @@ -86,7 +121,9 @@ class ToSql: elif analysis == 'touch_user_count': qry = sa.select(select_exprs + [func.count(sa.distinct(self.columns['#account_id'])).label('values')]) elif analysis == 'touch_user_avg': - qry = sa.select(select_exprs + [func.avg(self.columns['#account_id']).label('values')]) + qry = sa.select(select_exprs + [ + func.round((func.count() / func.count(sa.distinct(self.columns['#account_id']))), 2).label( + 'values')]) elif analysis == 'distinct_count': qry = sa.select( diff --git a/rbac_model.conf b/rbac_model.conf index 396ebe4..7a0ddc6 100644 --- a/rbac_model.conf +++ b/rbac_model.conf @@ -11,4 +11,4 @@ g = _, _, _ e = some(where (p.eft == allow)) [matchers] -m = (g(r.sub, p.sub) || g(r.sub, p.sub, r.dom)) && (p.dom=="*" || r.dom == p.dom) && ( p.obj=="*" || r.obj == p.obj) && (p.act=="*" || r.act == p.act) || r.sub=="root" \ No newline at end of file +m = (g(r.sub, p.sub) || g(r.sub, p.sub, r.dom) || keyMatch(r.sub, p.sub)) && (p.dom=="*" || r.dom == p.dom) && ( p.obj=="*" || r.obj == p.obj) && (p.act=="*" || r.act == p.act) || r.sub=="root" \ No newline at end of file diff --git a/schemas/project.py b/schemas/project.py index fd26650..17a83e4 100644 --- a/schemas/project.py +++ b/schemas/project.py @@ -13,6 +13,7 @@ class ProjectBase(BaseModel): class MemberRole(BaseModel): username: str + user_id: str role_name: str data_auth_id: str