From 46f640fcad48ad639942df997e0896c3141da453 Mon Sep 17 00:00:00 2001 From: wuaho Date: Thu, 26 Aug 2021 20:09:52 +0800 Subject: [PATCH] 1 --- api/api_v1/endpoints/dashboard.py | 14 +++++++ api/api_v1/endpoints/query.py | 70 ++++++++++++++++++++++++++++--- api/api_v1/endpoints/report.py | 3 ++ models/behavior_analysis.py | 16 ++++--- schemas/dashboard.py | 4 ++ utils/__init__.py | 1 + utils/export.py | 26 ++++++++++++ 7 files changed, 124 insertions(+), 10 deletions(-) create mode 100644 utils/export.py diff --git a/api/api_v1/endpoints/dashboard.py b/api/api_v1/endpoints/dashboard.py index d9c6783..a38dfac 100644 --- a/api/api_v1/endpoints/dashboard.py +++ b/api/api_v1/endpoints/dashboard.py @@ -129,6 +129,20 @@ async def add_report(data_in: schemas.AddReport, return schemas.Msg(code=0, msg='ok', data='ok') +@router.post("/edit_report") +async def edit_report(data_in: schemas.EditReport, + game: str, + db: AsyncIOMotorDatabase = Depends(get_database), + current_user: schemas.UserDB = Depends(deps.get_current_user) + ): + """编辑报表""" + + res = await crud.dashboard.update_one(db, {'_id': data_in.id, 'reports.report_id': data_in.report.report_id}, + {'$set': {'reports.$': data_in.report.dict()}}) + + return schemas.Msg(code=0, msg='ok', data='ok') + + @router.post("/del_report") async def del_report( game: str, diff --git a/api/api_v1/endpoints/query.py b/api/api_v1/endpoints/query.py index 8601000..a172374 100644 --- a/api/api_v1/endpoints/query.py +++ b/api/api_v1/endpoints/query.py @@ -1,10 +1,13 @@ import datetime from collections import defaultdict +import mimetypes +from urllib.parse import quote import pandas as pd import numpy as np from fastapi import APIRouter, Depends, Request from motor.motor_asyncio import AsyncIOMotorDatabase +from fastapi.responses import StreamingResponse import crud, schemas from common import * @@ -16,6 +19,7 @@ from db.redisdb import get_redis_pool, RedisDrive from models.behavior_analysis import BehaviorAnalysis from models.user_analysis import UserAnalysis +from utils import DfToStream router = APIRouter() @@ -46,6 +50,54 @@ async def event_model_sql( return schemas.Msg(code=0, msg='ok', data=data) +@router.post("/event_model_export") +async def event_model_export(request: Request, + game: str, + ckdb: CKDrive = Depends(get_ck_db), + analysis: BehaviorAnalysis = Depends(BehaviorAnalysis), + current_user: schemas.UserDB = Depends(deps.get_current_user) + ): + """ 事件分析模型 数据导出""" + await analysis.init() + sqls = analysis.event_model_sql() + file_name = quote(f'{sqls[0]["report_name"]}.xlsx') + mime = mimetypes.guess_type(file_name)[0] + excels = [] + for item in sqls: + sql = item['sql'] + event_name = item['event_name'] + df = await ckdb.query_dataframe(sql) + excels.append((df, event_name)) + df_to_stream = DfToStream(*excels) + with df_to_stream as d: + export = d.to_stream() + return StreamingResponse(export, media_type=mime, headers={'Content-Disposition': f'filename="{file_name}"'}) + + +# +# @router.get("/event_model_export") +# async def event_model_export(request: Request, +# game: str, +# report_id: str, +# ckdb: CKDrive = Depends(get_ck_db), +# # analysis: BehaviorAnalysis = Depends(BehaviorAnalysis), +# current_user: schemas.UserDB = Depends(deps.get_current_user) +# ): +# """ 事件分析模型 数据导出""" +# analysis = BehaviorAnalysis(game, schemas.CkQuery(report_id=report_id), get_redis_pool()) +# await analysis.init() +# sqls = analysis.event_model_sql() +# res = [] +# file_name = f'{sqls[0]["report_name"]}.xlsx' +# mime = mimetypes.guess_type(file_name)[0] +# for item in sqls[:1]: +# sql = item['sql'] +# event_name = item['event_name'] +# df = await ckdb.query_dataframe(sql) +# file = df_to_stream(df, event_name) +# return StreamingResponse(file, media_type=mime, headers={'Content-Disposition': f'filename="{file_name}"'}) +# + @router.post("/event_model") async def event_model( request: Request, @@ -83,7 +135,11 @@ async def event_model( q['groups'].append(groupby) q['values'].append(df['values'].to_list()) q['sum'].append(int(df['values'].sum())) - q['date_range'] = [f'{i[0]}-{i[1]}' for i in df.set_index(['svrindex', 'name']).index] + if groupby: + q['date_range'] = [f'{i}' for i in df.set_index(groupby).index] + else: + q['date_range'] =['合计'] + res.append(q) continue @@ -111,6 +167,10 @@ async def event_model( concat_data.append((i, 0)) df = pd.concat([df, pd.DataFrame(concat_data, columns=df.columns)]) df.sort_values('date', inplace=True) + if len(df) >= 2: + q['chain_ratio'] = round((df.iloc[-1, 1] - df.iloc[-2, 1]) * 100 / df.iloc[-2, 1], 2) + if len(df) >= 8: + q['wow'] = round((df.iloc[-1, 1] - df.iloc[-8, 1]) * 100 / df.iloc[-8, 1], 2) q['values'].append(df['values'].to_list()) q['sum'].append(int(df['values'].sum())) if item['time_particle'] in ('P1D', 'P1W'): @@ -157,7 +217,7 @@ async def retention_model(request: Request, for i in set(date_range) - set(df.index): df.loc[i] = 0 df.sort_index(inplace=True) - days = [i for i in range(unit_num+1)] + days = [i for i in range(unit_num + 1)] summary_values = {} today = datetime.datetime.today().date() for date, value in df.T.items(): @@ -167,11 +227,11 @@ async def retention_model(request: Request, tmp['n'] = [] tmp['p_outflow'] = [] tmp['n_outflow'] = [] - for i in range((today - date).days+1): + for i in range((today - date).days + 1): if i > unit_num: break - p = float(getattr(value, f'p{i+1}')) - n = int(getattr(value, f'cnt{i+1}')) + p = float(getattr(value, f'p{i + 1}')) + n = int(getattr(value, f'cnt{i + 1}')) p_outflow = round(100 - p, 2) n_outflow = value.cnt0 - n tmp['p'].append(p) diff --git a/api/api_v1/endpoints/report.py b/api/api_v1/endpoints/report.py index 7f3dd6a..cb0b84c 100644 --- a/api/api_v1/endpoints/report.py +++ b/api/api_v1/endpoints/report.py @@ -86,8 +86,11 @@ async def read_report( reports = await crud.report.read_report(db,project_id=data_in.project_id, projection=projection, **ext_where) + + for item in reports: item['added'] = False + # item['name'] = item['name'] item['show_config'] = dict() added_ids = {item['report_id']: item for item in dashboard.get('reports', [])} if item['_id'] in added_ids: diff --git a/models/behavior_analysis.py b/models/behavior_analysis.py index f7f6b44..b9ff05d 100644 --- a/models/behavior_analysis.py +++ b/models/behavior_analysis.py @@ -99,6 +99,7 @@ class BehaviorAnalysis: self.time_particle = None self.date_range = None self.unit_num = None + self.report_name = None async def init(self): if self.data_in.report_id: @@ -116,6 +117,8 @@ class BehaviorAnalysis: self.event_view['endTime'] = arrow.get().shift(days=-int(e_days)).strftime('%Y-%m-%d 23:59:59') self.event_view['startTime'] = arrow.get().shift(days=-int(s_days)).strftime('%Y-%m-%d 00:00:00') + self.report_name = report["name"] + else: self.event_view = self.data_in.eventView @@ -360,7 +363,8 @@ class BehaviorAnalysis: 'date_range': self.date_range, 'event_name': event_name_display or event_name, 'format': format, - 'time_particle': self.time_particle + 'time_particle': self.time_particle, + 'report_name': self.report_name or 'temp' }) return sqls @@ -435,8 +439,10 @@ ORDER BY level quota_interval_arr = event.get('quotaIntervalArr') where = [ - event_date_col >= self.start_date, - event_date_col <= self.end_date, + # event_date_col >= self.start_date, + # event_date_col <= self.end_date, + func.addHours(event_time_col, self.zone_time) >= self.start_date, + func.addHours(event_time_col, self.zone_time) <= self.end_date, event_name_col == event_name ] event_filter, _ = self.handler_filts(*self.events[0]['filts']) @@ -611,7 +617,7 @@ ORDER BY values desc""" days = (arrow.get(self.end_date).date() - arrow.get(self.start_date).date()).days keep = [] cnt = [] - for i in range(days+1): + for i in range(days + 1): keep.append( f"""cnt{i + 1},round(cnt{i + 1} * 100 / cnt0, 2) as `p{i + 1}`""") cnt.append(f"""sum(if(dateDiff('day',a.reg_date,b.visit_date)={i},1,0)) as cnt{i + 1}""") @@ -643,6 +649,6 @@ group by a.reg_date) log on reg.date=log.reg_date print(sql) return { 'sql': sql, - 'date_range':self.date_range, + 'date_range': self.date_range, 'unit_num': self.unit_num } diff --git a/schemas/dashboard.py b/schemas/dashboard.py index aaa9f42..49794f7 100644 --- a/schemas/dashboard.py +++ b/schemas/dashboard.py @@ -71,6 +71,10 @@ class DelReport(DBBase): report_id: str +class EditReport(DBBase): + report: Report + + # -------------------------------------------------------------- # 数据库模型 class DashboardDB(DBBase): diff --git a/utils/__init__.py b/utils/__init__.py index 392b3c8..c6019dd 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -2,3 +2,4 @@ from .adapter import * from . import casbin from .func import * +from .export import * diff --git a/utils/export.py b/utils/export.py new file mode 100644 index 0000000..c2a3938 --- /dev/null +++ b/utils/export.py @@ -0,0 +1,26 @@ +from io import StringIO, BytesIO +import pandas as pd + + +class DfToStream: + def __init__(self, *args, index=False, **kwargs): + self.dfs = args + self.index = index + self.writer = None + self.output = None + + def __enter__(self): + self.output = BytesIO() + self.writer = pd.ExcelWriter(self.output, engine='xlsxwriter') + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.writer.close() + self.output.seek(0) + + def to_stream(self): + for item in self.dfs: + df = item[0] + sheet_name = item[1] + df.to_excel(self.writer, encoding='utf-8', sheet_name=sheet_name, index=False) + return self.output