diff --git a/config.json b/config.json index eea850a..2278e5a 100644 --- a/config.json +++ b/config.json @@ -9,9 +9,15 @@ "dest_coll": "user", "task_name": "first_recharge" }, - "repair_gunfu": { + "repair_gunfu": { "source_coll": "user", "dest_coll": "user", "task_name": "repair_gunfu" + }, + "summary_login": { + "source_coll": "event", + "dest_coll": "summary_login", + "task_name": "summary_login" } + } \ No newline at end of file diff --git a/task/first_recharge.py b/task/first_recharge.py index e1bcfe2..7dc30c8 100644 --- a/task/first_recharge.py +++ b/task/first_recharge.py @@ -55,6 +55,6 @@ class FirstRecharge(Task): logger.error(f'msg:{e}') # pass if bulk_data: - self.remote_db[self.dest_coll].bulk_write(bulk_data, ordered=False) self.local_db[self.dest_coll].bulk_write(bulk_data, ordered=False) + self.remote_db[self.dest_coll].bulk_write(bulk_data, ordered=False) self.set_cursor(cursor_st=ts['cursor_st'], cursor_et=ts['cursor_et']) diff --git a/task/repair_gunfu.py b/task/repair_gunfu.py index 660ecba..3bd162f 100644 --- a/task/repair_gunfu.py +++ b/task/repair_gunfu.py @@ -4,7 +4,6 @@ from pydantic import BaseModel, Field from .task import Task from utils import * -from model import MdbObjectId class RepairGunfu(Task): @@ -13,7 +12,7 @@ class RepairGunfu(Task): """ class Model(BaseModel): - id: MdbObjectId = Field(..., title='_id', alias='_id') + game_role_id: str = Field(None, title="角色id", alias='_game_role_id') device_id: str = Field(..., title='设备id', alias='_device_id') @classmethod @@ -47,12 +46,12 @@ class RepairGunfu(Task): {'_device_id': device_id, 'gunfu_num': {'$exists': True}}) bulk_data.append( - UpdateOne({'_id': model.id}, + UpdateOne({'_game_role_id': model.game_role_id}, {'$set': {'gunfu_num': role_cnt + 1}}, upsert=True)) except Exception as e: logger.error(f'msg:{e}') # pass if bulk_data: - self.remote_db[self.dest_coll].bulk_write(bulk_data, ordered=False) self.local_db[self.dest_coll].bulk_write(bulk_data, ordered=False) + self.remote_db[self.dest_coll].bulk_write(bulk_data, ordered=False) self.set_cursor(cursor_st=ts['cursor_st'], cursor_et=ts['cursor_et']) diff --git a/task/summary_func.py b/task/summary_func.py index f43c17b..32bcad2 100644 --- a/task/summary_func.py +++ b/task/summary_func.py @@ -46,5 +46,6 @@ class SummaryFunc(Task): logger.error(f'ftype {item["ftype"]} msg:{e}') # pass if bulk_data: + self.local_db[self.dest_coll].bulk_write(bulk_data, ordered=False) self.remote_db[self.dest_coll].bulk_write(bulk_data, ordered=False) self.set_cursor(cursor_st=ts['cursor_st'], cursor_et=ts['cursor_et']) diff --git a/task/summary_login.py b/task/summary_login.py new file mode 100644 index 0000000..4f79e58 --- /dev/null +++ b/task/summary_login.py @@ -0,0 +1,64 @@ +from pymongo import UpdateOne +from pydantic import Field +import pandas as pd + +from .task import Task +from utils import * +from model import GBaseModel + + +class SummaryLogin(Task): + """ + 每日活跃 + """ + + class Model(GBaseModel): + cdate: int = Field(..., title='当天0点') + # first_device_id: str = Field(None, title='第一次登录设备id', alias='_first_device_id') + manufacturer: str = Field(None, title='设备品牌', alias='_manufacturer') + model: str = Field(None, title='型号', alias='_model') + os_version: str = Field(None, title='系统版本', alias='_os_version') + app_name: str = Field(None, title='游戏版本', alias='_app_name') + screen_width: int = Field(None, title='屏幕宽', alias='_screen_width') + screen_height: int = Field(None, title='屏幕高', alias='_screen_height') + + def cleaning(self, cursor_list): + for cursor in cursor_list: # type:dict + for source_coll, ts in cursor.items(): # type:str,dict + if ts['cursor_st'] == ts['cursor_et']: + continue + logger.info(f'开始处理{self.game_name} 处理 {source_coll} 游标 {ts}') + where = { + '_event_time': { + '$gte': ts['cursor_st'], + '$lt': ts['cursor_et'], + }, + '_event_name': 'TimeSpending' # 在线时长打点 30s + } + + projection = self.Model.get_fields() + bulk_data = [] + # 当前活跃role id + role_list = self.local_db[source_coll].distinct('_game_role_id', where) + # 当日已记录的活跃role + cdate = int(pd.Timestamp(ts['cursor_st'], unit='s', tz=self.timezone) \ + .normalize().timestamp()) + exists_role_list = self.local_db[self.dest_coll].distinct('_game_role_id', {'cdate': cdate}) + + # 还没有记录的 + role_set = set(role_list) - set(exists_role_list) + for role_id in role_set: + item = self.local_db[source_coll].find_one( + {'_game_role_id': role_id, '_event_name': 'TimeSpending'}, projection) + try: + item['cdate'] = cdate + model = self.Model(**item) + data = model.dict(by_alias=True) + bulk_data.append(UpdateOne({'_id': data['_id']}, {'$set': data}, upsert=True)) + except Exception as e: + logger.error(f'msg:{e}') + # pass + if bulk_data: + self.local_db[self.dest_coll].bulk_write(bulk_data, ordered=False) + self.remote_db[self.dest_coll].bulk_write(bulk_data, ordered=False) + self.set_cursor(cursor_st=ts['cursor_st'], cursor_et=ts['cursor_et'])