From c2993483d845e9a7694e3bc556b23e100561d9d7 Mon Sep 17 00:00:00 2001 From: wuaho Date: Thu, 29 Jul 2021 19:21:52 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E7=95=99=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/api_v1/endpoints/data_auth.py | 18 ++++++++---------- api/api_v1/endpoints/query.py | 18 +++++++++--------- models/behavior_analysis.py | 29 ++++++++++++++++++++++------- 3 files changed, 39 insertions(+), 26 deletions(-) diff --git a/api/api_v1/endpoints/data_auth.py b/api/api_v1/endpoints/data_auth.py index f50ea31..b6ffc24 100644 --- a/api/api_v1/endpoints/data_auth.py +++ b/api/api_v1/endpoints/data_auth.py @@ -43,6 +43,7 @@ async def edit_data_auth(request: Request, await crud.data_auth.edit_data_auth(db, data_id) return schemas.Msg(code=0, msg='ok', data=data_id) + @router.get("/quotas_map") async def quotas_map( request: Request, @@ -103,7 +104,7 @@ async def my_event(request: Request, current_user: schemas.UserDB = Depends(deps.get_current_user) ) -> schemas.Msg: """获取自己的事件权限""" - + event_list = [] data_auth_id = await crud.authority.get_data_auth_id(db, game, request.user.username) my_data_auth = [] @@ -112,6 +113,7 @@ async def my_event(request: Request, if data_auth_id == '*': my_data_auth = await ck.distinct(game, 'event', '#event_name') + else: # 设置了数据权限 my_data_auth = await crud.data_auth.get(db, ObjectId(data_auth_id)) @@ -120,8 +122,6 @@ async def my_event(request: Request, if not my_data_auth: return schemas.Msg(code=0, msg='ok', data=[]) - event_list = [] - event_show_name = await crud.event_mana.get_all_show_name(db, game) event_list.append({'id': 'event', 'title': '全部事件', 'category': []}) for item in my_data_auth: @@ -129,6 +129,7 @@ async def my_event(request: Request, 'event_name': item, 'event_desc': event_show_name.get(item, item) }) + event_list[-1]['category'].append({'event_name': '*', 'event_desc': '任意事件'}) event_list.sort() return schemas.Msg(code=0, msg='ok', data=event_list) @@ -146,7 +147,7 @@ async def user_property(request: Request, data = json.loads(data) propertys = [] - data_attr = await crud.data_attr.find_many(db, {'game':game, 'cat':'user'}) + data_attr = await crud.data_attr.find_many(db, {'game': game, 'cat': 'user'}) data_attr = {item['name']: item for item in data_attr} for k, v in data.items(): @@ -162,9 +163,6 @@ async def user_property(request: Request, return schemas.Msg(code=0, msg='ok', data=propertys) - - - @router.post('/load_prop_quotas') async def load_prop_quotas(request: Request, game: str, @@ -181,7 +179,7 @@ async def load_prop_quotas(request: Request, all_filed = await rdb.get(f'{game}_event') all_filed = json.loads(all_filed) - data_attr = await crud.data_attr.find_many(db, {'game':game, 'cat':'event'}) + data_attr = await crud.data_attr.find_many(db, {'game': game, 'cat': 'event'}) data_attr = {item['name']: item for item in data_attr} event_props = [] for item in event_prop_list: @@ -274,7 +272,7 @@ async def load_filter_props(request: Request, all_filed = await rdb.get(f'{game}_event') all_filed = json.loads(all_filed) - data_attr = await crud.data_attr.find_many(db, {'game':game, 'cat':'event'}) + data_attr = await crud.data_attr.find_many(db, {'game': game, 'cat': 'event'}) data_attr = {item['name']: item for item in data_attr} event_props = [] for item in event_prop_list: @@ -288,7 +286,7 @@ async def load_filter_props(request: Request, } event_props.append(event_prop) - data_attr = await crud.data_attr.find_many(db, {'game':game, 'cat':'user'}) + data_attr = await crud.data_attr.find_many(db, {'game': game, 'cat': 'user'}) data_attr = {item['name']: item for item in data_attr} user_props = [] for item in user_prop_list: diff --git a/api/api_v1/endpoints/query.py b/api/api_v1/endpoints/query.py index c481bb8..04e1a3f 100644 --- a/api/api_v1/endpoints/query.py +++ b/api/api_v1/endpoints/query.py @@ -72,7 +72,7 @@ async def event_model( q['date_range'] = date_range df = await ckdb.query_dataframe(sql) if df.shape[0] == 0: - return schemas.Msg(code=0, msg='ok', data=q) + return schemas.Msg(code=0, msg='ok', data=[q]) if groupby: # 有分组 @@ -141,31 +141,31 @@ async def retention_model( concat_data = [] df = pd.concat([df, pd.DataFrame(concat_data, columns=df.columns)]) # 计算整体 - summary_df = df.groupby(['date', 'event_name'])[['values', 'amount']].sum() + summary_df = df.groupby('date')[['val_a', 'val_b', 'amount_a']].sum() summary_values = {} for i, d1 in enumerate(date_range): - a = set(summary_df.loc[(d1, event_a)]['values']) if (d1, event_a) in summary_df.index else set() + a = set(summary_df.loc[d1]['val_a']) if d1 in summary_df.index else set() if not a: continue key = d1.strftime('%Y-%m-%d') for j, d2 in enumerate(date_range[i:]): if j > unit_num: break - b = set(summary_df.loc[(d2, event_b)]['values']) if (d2, event_b) in summary_df.index else set() + b = set(summary_df.loc[d2]['val_b']) if d2 in summary_df.index else set() tmp = summary_values.setdefault(key, {}) tmp.setdefault('d0', len(a)) tmp.setdefault('p', []).append(division(len(a & b) * 100, len(a))) tmp.setdefault('n', []).append(len(a & b)) - groups = set([tuple(i) for i in df[res['groupby'][2:]].values]) + groups = set([tuple(i) for i in df[res['groupby']].values]) df.set_index(res['groupby'], inplace=True) df.sort_index(inplace=True) values = {} days = [i for i in range((date_range[-1] - date_range[0]).days + 1)][:unit_num + 1] for i, d1 in enumerate(date_range): for g in groups: - if g == tuple(): + if len(g) == 1: continue - a = set(df.loc[(d1, event_a, *g)]['values']) if (d1, event_a, *g) in df.index else set() + a = set(df.loc[g]['val_a']) if g in df.index else set() if not a: continue key = d1.strftime("%Y-%m-%d") @@ -173,8 +173,8 @@ async def retention_model( for j, d2 in enumerate(date_range[i:]): if j > unit_num: break - b = set(df.loc[(d2, event_b, *g)]['values']) if (d2, event_b, *g) in df.index else set() - tmp = tmp_g.setdefault(','.join(g), {}) + b = set(df.loc[g]['val_b']) if g in df.index else set() + tmp = tmp_g.setdefault(','.join(g[1:]), {}) tmp.setdefault('d0', len(a)) tmp.setdefault('p', []).append(division(len(a & b) * 100, len(a))) tmp.setdefault('n', []).append(len(a & b)) diff --git a/models/behavior_analysis.py b/models/behavior_analysis.py index 1d77c88..5a9bc71 100644 --- a/models/behavior_analysis.py +++ b/models/behavior_analysis.py @@ -44,7 +44,7 @@ class BehaviorAnalysis: self.event_view = report['query']['eventView'] self.events = report['query']['events'] e_days, s_days = self.event_view['recentDay'].split('-') - self.event_view['endTime'] = arrow.get().shift(days=-int(e_days)+1).strftime('%Y-%m-%d 23:59:59') + self.event_view['endTime'] = arrow.get().shift(days=-int(e_days) + 1).strftime('%Y-%m-%d 23:59:59') self.event_view['startTime'] = arrow.get().shift(days=-int(s_days)).strftime('%Y-%m-%d 00:00:00') @@ -144,27 +144,42 @@ class BehaviorAnalysis: def retention_model_sql(self): event_name_a = self.events[0]['eventName'] event_name_b = self.events[1]['eventName'] + visit_name = self.events[0].get('event_attr_id') event_time_col = getattr(self.event_tbl.c, '#event_time') event_name_col = getattr(self.event_tbl.c, '#event_name') e_account_id_col = getattr(self.event_tbl.c, '#account_id') u_account_id_col = getattr(self.user_tbl.c, '#account_id') date_col = sa.Column('date') + who_visit = e_account_id_col + if visit_name: + who_visit = getattr(self.event_tbl.c, visit_name) selectd = [func.toStartOfDay(func.addHours(event_time_col, self.zone_time)).label('date'), - event_name_col.label('event_name'), *self.groupby, - func.arrayDistinct(func.groupArray(e_account_id_col)).label('values'), - func.length(sa.Column('values')).label('amount') + func.arrayDistinct( + (func.groupArray(func.if_(event_name_col == event_name_a, who_visit, None)))).label('val_a'), + + func.length(sa.Column('val_a')).label('amount_a'), + func.length(sa.Column('val_b')).label('amount_b'), ] + + if event_name_b == '*': + val_b = func.arrayDistinct( + (func.groupArray(func.if_(event_name_col != event_name_a, who_visit, None)))).label('val_b'), + selectd.insert(-2, *val_b) + else: + val_b = func.arrayDistinct( + (func.groupArray(func.if_(event_name_col == event_name_b, who_visit, None)))).label('val_b'), + selectd.insert(-2, *val_b) + base_where = [ func.addHours(event_time_col, self.zone_time) >= self.start_date, func.addHours(event_time_col, self.zone_time) <= self.end_date, - event_name_col.in_([event_name_a, event_name_b]), ] event_filter, user_filter = self.handler_filts() - groupby = [date_col, event_name_col] + self.groupby + groupby = [date_col] + self.groupby oredrby = [date_col] if user_filter: qry = sa.select(selectd).select_from( @@ -177,7 +192,7 @@ class BehaviorAnalysis: sql = str(qry.compile(compile_kwargs={"literal_binds": True})) print(sql) return {'sql': sql, - 'groupby': ['date', 'event_name'] + [i.key for i in self.groupby], + 'groupby': ['date'] + [i.key for i in self.groupby], 'date_range': self.date_range, 'event_name': [event_name_a, event_name_b], 'unit_num': self.unit_num