diff --git a/config.json b/config.json index 7fd61b7..7ef2f38 100644 --- a/config.json +++ b/config.json @@ -28,6 +28,10 @@ "source_coll": "paylist", "dest_coll": "summary_pay", "task_name": "summary_pay" + }, + "summary_shopbuy": { + "source_coll": "event", + "dest_coll": "summary_shopbuy", + "task_name": "summary_shopbuy" } - } \ No newline at end of file diff --git a/task/summary_func.py b/task/summary_func.py index 32bcad2..f805c8f 100644 --- a/task/summary_func.py +++ b/task/summary_func.py @@ -35,10 +35,10 @@ class SummaryFunc(Task): projection = self.Model.get_fields() bulk_data = [] + cdate = int(pd.Timestamp(ts['cursor_st'], unit='s', tz=self.timezone).normalize().timestamp()) for item in self.local_db[source_coll].find(where, projection): try: - item['cdate'] = int(pd.Timestamp(item['_event_time'], unit='s', tz=self.timezone) \ - .normalize().timestamp()) + item['cdate'] = cdate model = self.Model(**item) data = model.dict(by_alias=True) bulk_data.append(UpdateOne({'_id': data['_id']}, {'$set': data}, upsert=True)) diff --git a/task/summary_shopbuy.py b/task/summary_shopbuy.py new file mode 100644 index 0000000..70f3dce --- /dev/null +++ b/task/summary_shopbuy.py @@ -0,0 +1,66 @@ +from typing import List + +from pymongo import UpdateOne +from pydantic import Field +import pandas as pd + +from .task import Task +from utils import * +from model import GBaseModel, IntStr + + +class SummaryShopbuy(Task): + """ + 功能分析 + """ + + class Model(GBaseModel): + cdate: int = Field(..., title='当天0点') + prize: List[dict] = Field(None, title='奖励') + need: List[dict] = Field(None, title='消耗') + user_name: str = Field(None, title='昵称') + stype: IntStr = Field(None, title='商店id') + needa: str = Field(...) + needn: int = Field(...) + needt: str = Field(...) + prizea: str = Field(...) + prizen: int = Field(...) + prizet: str = Field(...) + + def cleaning(self, cursor_list): + for cursor in cursor_list: # type:dict + for source_coll, ts in cursor.items(): # type:str,dict + if ts['cursor_st'] == ts['cursor_et']: + continue + logger.info(f'开始处理{self.game_name} 处理 {source_coll} 游标 {ts}') + where = { + '_event_name': 'Shop', + "act": "buy", + '_event_time': { + '$gte': ts['cursor_st'], + '$lt': ts['cursor_et'], + } + } + + projection = self.Model.get_fields() + bulk_data = [] + cdate = int(pd.Timestamp(ts['cursor_st'], unit='s', tz=self.timezone).normalize().timestamp()) + for item in self.local_db[source_coll].find(where, projection): + try: + item['cdate'] = cdate + item['needa'] = item['need'][0]['a'] + item['needt'] = item['need'][0]['t'] + item['needn'] = item['need'][0]['n'] + item['prizea'] = item['prize'][0]['a'] + item['prizet'] = item['prize'][0]['t'] + item['prizen'] = item['prize'][0]['n'] + model = self.Model(**item) + data = model.dict(by_alias=True) + bulk_data.append(UpdateOne({'_id': data['_id']}, {'$set': data}, upsert=True)) + except Exception as e: + logger.error(f'msg:{e}') + # pass + if bulk_data: + self.local_db[self.dest_coll].bulk_write(bulk_data, ordered=False) + self.remote_db[self.dest_coll].bulk_write(bulk_data, ordered=False) + self.set_cursor(cursor_st=ts['cursor_st'], cursor_et=ts['cursor_et'])