From 2950cf264804df0289b5a9fd2df0541ff2fc7e3b Mon Sep 17 00:00:00 2001 From: wuaho Date: Thu, 22 Jul 2021 11:14:15 +0800 Subject: [PATCH] update --- api/api_v1/api.py | 2 + api/api_v1/endpoints/query.py | 2 +- api/api_v1/endpoints/xquery.py | 32 ++++++++++++++ models/x_analysis.py | 76 ++++++++++++++++++++++++++++++++++ schemas/__init__.py | 1 + schemas/xquery.py | 13 ++++++ sql/总览.sql | 57 +++++++++++++++++++++++++ sql/新增付费.sql | 17 ++++++++ sql/新增用户.sql | 4 -- sql/活跃用户.sql | 4 -- 10 files changed, 199 insertions(+), 9 deletions(-) create mode 100644 api/api_v1/endpoints/xquery.py create mode 100644 models/x_analysis.py create mode 100644 schemas/xquery.py create mode 100644 sql/总览.sql create mode 100644 sql/新增付费.sql delete mode 100644 sql/新增用户.sql delete mode 100644 sql/活跃用户.sql diff --git a/api/api_v1/api.py b/api/api_v1/api.py index f12c669..04674b9 100644 --- a/api/api_v1/api.py +++ b/api/api_v1/api.py @@ -8,6 +8,7 @@ from .endpoints import report from .endpoints import authority from .endpoints import data_mana from .endpoints import query +from .endpoints import xquery from .endpoints import data_auth from .endpoints import event_mana from .endpoints import test @@ -29,3 +30,4 @@ api_router.include_router(data_mana.router, tags=["数据管理"], prefix='/data api_router.include_router(event_mana.router, tags=["数据管理"], prefix='/data_mana') api_router.include_router(query.router, tags=["ck"], prefix='/ck') +api_router.include_router(xquery.router, tags=["xck"], prefix='/ck') diff --git a/api/api_v1/endpoints/query.py b/api/api_v1/endpoints/query.py index 584398b..48b10b2 100644 --- a/api/api_v1/endpoints/query.py +++ b/api/api_v1/endpoints/query.py @@ -506,7 +506,7 @@ async def user_property_model( title = ['总体', quota] else: - sum_s = df.groupby(groupby)['values'].sum().sort_values(ascending=False) + sum_s = df.groupby(groupby)['values'].sum() data = dict() for key, val in sum_s.items(): if isinstance(key, tuple): diff --git a/api/api_v1/endpoints/xquery.py b/api/api_v1/endpoints/xquery.py new file mode 100644 index 0000000..4304b21 --- /dev/null +++ b/api/api_v1/endpoints/xquery.py @@ -0,0 +1,32 @@ +from collections import defaultdict + +import pandas as pd +import numpy as np +from fastapi import APIRouter, Depends, Request +from motor.motor_asyncio import AsyncIOMotorDatabase + +import crud, schemas +from common import * + +from api import deps +from db import get_database +from db.ckdb import get_ck_db, CKDrive +from db.redisdb import get_redis_pool, RedisDrive + +from models.behavior_analysis import BehaviorAnalysis +from models.user_analysis import UserAnalysis +from models.x_analysis import XAnalysis + +router = APIRouter() + + +@router.post("/overview") +async def event_model_sql( + request: Request, + game: str, + analysis: XAnalysis = Depends(XAnalysis), + current_user: schemas.UserDB = Depends(deps.get_current_user) +) -> schemas.Msg: + """ 数据总览 """ + sql = analysis.to_sql() + return schemas.Msg(code=0, msg='ok', data=sql) diff --git a/models/x_analysis.py b/models/x_analysis.py new file mode 100644 index 0000000..3317bea --- /dev/null +++ b/models/x_analysis.py @@ -0,0 +1,76 @@ +from typing import Tuple + +import sqlalchemy as sa +import json + +from fastapi import Depends + +import pandas as pd + +from sqlalchemy import func, or_, and_, not_, MetaData + +import schemas +from core.config import settings +from db.redisdb import get_redis_pool, RedisDrive + + +class XAnalysis: + def __init__(self, data_in: schemas.Overview, game: str): + self.data_in = data_in + self.game = game + + def to_sql(self): + # 构建表 + metadata = sa.MetaData(schema=self.game) + tbl = sa.Table(f'overview', metadata, + sa.Column('date'), + sa.Column('#os'), + sa.Column('#bundle_id'), + sa.Column('owner_name'), + sa.Column('channel'), + sa.Column('active_num'), + sa.Column('new_account_num'), + sa.Column('money'), + sa.Column('pay_account_num'), + sa.Column('new_pay_account_num'), + sa.Column('new_money'), + ) + # bundle_id_col = sa.Column('#bundle_id') + # channel_col = sa.Column('channel') + # owner_name_col = sa.Column('owner_name') + # os_col = sa.Column('#os') + # date_col = sa.Column('date') + # active_num_col = sa.Column('active_num') + # new_account_num_col = sa.Column('new_account_num') + + where = [ + getattr(tbl.c, 'date') >= self.data_in.sdate, + getattr(tbl.c, 'date') <= self.data_in.edate, + ] + + if self.data_in.bundle_id: + where.append(getattr(tbl.c, '#bundle_id').in_(self.data_in.bundle_id)) + + if self.data_in.channel: + where.append(getattr(tbl.c, '#bundle_id').in_(self.data_in.channel)) + + if self.data_in.owner_name: + where.append(getattr(tbl.c, 'owner_name').in_(self.data_in.owner_name)) + + if self.data_in.os: + where.append(getattr(tbl.c, '#os').in_(self.data_in.os)) + + qry = sa.select(getattr(tbl.c, 'date'), getattr(tbl.c, '#bundle_id'), + func.sum(getattr(tbl.c, 'active_num')).label('active_num'), + func.sum(getattr(tbl.c, 'new_account_num')).label('new_account_num'), + func.sum(getattr(tbl.c, 'pay_account_num')).label('pay_account_num'), + func.sum(getattr(tbl.c, 'money')).label('money'), + func.sum(getattr(tbl.c, 'new_pay_account_num')).label('new_pay_account_num'), + func.sum(getattr(tbl.c, 'new_money')).label('new_money'), + + ) \ + .where(*where) \ + .group_by(getattr(tbl.c, 'date'), getattr(tbl.c, '#bundle_id')) + sql = str(qry.compile(compile_kwargs={"literal_binds": True})) + print(sql) + return sql diff --git a/schemas/__init__.py b/schemas/__init__.py index ab25a2e..918d2c7 100644 --- a/schemas/__init__.py +++ b/schemas/__init__.py @@ -12,3 +12,4 @@ from .data_attr import * from .sql import * from .api_log import * from .event_mana import * +from .xquery import * \ No newline at end of file diff --git a/schemas/xquery.py b/schemas/xquery.py new file mode 100644 index 0000000..07476ea --- /dev/null +++ b/schemas/xquery.py @@ -0,0 +1,13 @@ +from typing import Any, List +from datetime import date +from pydantic import BaseModel + + +class Overview(BaseModel): + cat: str = 'account' + bundle_id: List[str] = None + os: List[str] = None + sdate: date + edate: date + owner_name: List[str] = None + channel: List[str] = None diff --git a/sql/总览.sql b/sql/总览.sql new file mode 100644 index 0000000..60c6b8f --- /dev/null +++ b/sql/总览.sql @@ -0,0 +1,57 @@ +with 'Android' as os, + 'gmhdtt' as owner, + '2021-07-11' as start_date, + '2021-07-20' as end_date +select date, + `#bundle_id`, + active_num, + new_account_num, + new_account_array, + money, + recharge_account_num, + round(recharge_account_num * 100 / active_num, 2) as pay_rate, + round(money / recharge_account_num, 2) as arppu, + round(money / active_num, 2) as arpu, + arrayMap(x-> x.2, arrayFilter(x->has(new_account_array, x.1), bid_money)) as new_recharge, + arraySum(new_recharge) as new_money, + length(new_recharge) as new_pay_num, + round(new_pay_num*100/new_account_num,2) as new_pay_rate, + round(new_money*100/new_pay_num,2) as new_arppu, + round(new_money*100/new_account_num,2) as new_arpu +from (select date, + `#bundle_id`, + active_num, + new_account_num, + new_account_array, + money, + recharge_account_num + from (select date, `#bundle_id`, active_num, new_account_num, new_account_array + from (select date, `#bundle_id`, sum(num) as active_num + from zhengba.active_account + where date >= start_date + and date <= end_date + and `#os` = os + and owner_name = owner + group by date, `#bundle_id`) as active_tbl + left join (select date, + `#bundle_id`, + sum(num) as new_account_num, + flatten(groupArray(account)) as new_account_array + from zhengba.new_account + where date >= start_date + and date <= end_date + and `#os` = os + and owner_name = owner + group by date, `#bundle_id`) as new_account_tbl + on active_tbl.date = new_account_tbl.date and + active_tbl.`#bundle_id` = new_account_tbl.`#bundle_id`) as tb1 + left join (select date, `#bundle_id`, sum(money) as money, sum(account_num) as recharge_account_num + from zhengba.recharge_game + where date >= start_date + and date <= end_date + and `#os` = os + and owner_name = owner + group by date, `#bundle_id`) as recharge_tbl + on recharge_tbl.date = tb1.date and tb1.`#bundle_id` = recharge_tbl.`#bundle_id`) as tb2 + left join zhengba.new_account_recharge as tb3 on tb2.date = tb3.date +order by date desc diff --git a/sql/新增付费.sql b/sql/新增付费.sql new file mode 100644 index 0000000..c7f7351 --- /dev/null +++ b/sql/新增付费.sql @@ -0,0 +1,17 @@ +create view new_account_recharge as (select date, + arrayMap((x, y) -> (x, y),groupArray(binduid), groupArray(money)) as bid_money +from (select date, binduid, money + from (select date, account, binduid, money + from (SELECT toDate(addHours(`#event_time`, `#zone_offset`)) AS date, + arrayDistinct(groupArray(binduid)) AS account + FROM zhengba.event + WHERE role_idx = 1 + GROUP BY toDate(addHours(`#event_time`, `#zone_offset`))) as tb1 + left join (select toDate(addHours(`#event_time`, `#zone_offset`)) as date, + binduid, + sum(money) as money + from zhengba.event + where `#event_name` = 'rechargeGame' + group by toDate(addHours(`#event_time`, `#zone_offset`)), binduid ) as tb2 + on tb1.date = tb2.date) + where has(account, binduid)) group by date) \ No newline at end of file diff --git a/sql/新增用户.sql b/sql/新增用户.sql deleted file mode 100644 index dd535e6..0000000 --- a/sql/新增用户.sql +++ /dev/null @@ -1,4 +0,0 @@ -select toDate(addHours(`#event_time`,8)), groupArray(`binduid`) as account,length(account) as num -from zhengba.event -where role_idx = 1 -group by toDate(addHours(`#event_time`,8)) \ No newline at end of file diff --git a/sql/活跃用户.sql b/sql/活跃用户.sql deleted file mode 100644 index 09d63b9..0000000 --- a/sql/活跃用户.sql +++ /dev/null @@ -1,4 +0,0 @@ -select toDate(addHours(`#event_time`, 8)), - uniqCombined(binduid), groupArray(distinct binduid) as bids -from zhengba.event -group by toDate(addHours(`#event_time`, 8)) \ No newline at end of file