config.json

@@ -28,15 +28,6 @@
     "source_coll": "paylist",
     "dest_coll": "summary_pay",
     "task_name": "summary_pay"
-  },
-  "summary_shopbuy": {
-    "source_coll": "event",
-    "dest_coll": "summary_shopbuy",
-    "task_name": "summary_shopbuy"
-  },
-  "owner_channel_server": {
-    "source_coll": "user",
-    "dest_coll": "attr",
-    "task_name": "owner_channel_server"
   }
+
 }

task/owner_channel_server.py

@@ -1,128 +0,0 @@
-import requests
-from pydantic import BaseModel, Field
-
-from db import get_local_db
-from .task import Task
-from utils import *
-
-
-class OwnerChannelServer(Task):
-    """
-    owner_channel_server
-    """
-
-    class Model(BaseModel):
-        channel_name: str = Field(..., min_length=1, title="channel", alias='_channel_name')
-        owner_name: str = Field(..., min_length=1, title="owner", alias='_owner_name')
-        district_server_id: int = Field(..., title="区服id", alias='_district_server_id')
-
-        @classmethod
-        def get_fields(cls):
-            return [v.alias for v in cls.__fields__.values()]
-
-    def get_channel_alias(self):
-        admin_game_db = get_local_db('admin_game')
-        res = admin_game_db['game'].find_one({'id_name': self.game_name}, {'server_name': True}) or {}
-        server_name = res.get('server_name')
-        if not server_name:
-            raise ValueError(f'{self.game_name} 请补充对应gm平台的游戏代号')
-        url = f'http://gamemana.legu.cc/index.php?g=admin&m=public&a=channel_json&where_select=game&where_input={server_name}&skey=legukey'
-        resp = requests.get(url)
-        if resp.status_code != 200:
-            return {}
-        data = resp.json()
-        channel_map = {}
-        for channel in data['data']:
-            channel_map[channel['id']] = channel['alias'] or channel['channel']
-            channel_map[channel['channel']] = channel['alias'] or channel['channel']
-        return channel_map
-
-    def get_server_alias(self):
-        admin_game_db = get_local_db('admin_game')
-        res = admin_game_db['game'].find_one({'id_name': self.game_name}, {'server_name': True}) or {}
-        server_name = res.get('server_name')
-        if not server_name:
-            raise ValueError(f'{self.game_name} 请补充对应gm平台的游戏代号')
-        url = f'http://gametools.legu.cc/?app=api&act=getServerList&game={server_name}'
-        resp = requests.get(url)
-        if resp.status_code != 200:
-            return {}
-        return {server['serverid']: server['servername'] for server in resp.json()}
-
-    def cleaning(self, cursor_list):
-        for cursor in cursor_list:  # type:dict
-            for source_coll, ts in cursor.items():  # type:str,dict
-                if ts['cursor_st'] == ts['cursor_et']:
-                    continue
-                logger.info(f'开始处理{self.game_name} 处理 {source_coll} 游标 {ts}')
-                where = {
-                    'role_create_time': {
-                        '$gte': ts['cursor_st'],
-                        '$lt': ts['cursor_et'],
-                    }
-                }
-
-                projection = self.Model.get_fields()
-                owner_channel_server_data = {}
-                for item in self.local_db[source_coll].find(where, projection):
-                    try:
-                        model = self.Model(**item)
-                        channel_dict = owner_channel_server_data.setdefault(model.owner_name, {})
-                        server_set = channel_dict.setdefault(model.channel_name, set())
-                        server_set.add(model.district_server_id)
-
-                    except Exception as e:
-                        logger.error(f'msg:{e}')
-                        continue
-                        # pass
-                select_map_data = (self.local_db['attr'].find_one({'pname': 'select_map'}) or {}).get('data', {})
-                # 将 owner_channel_server_data 合并到 select_map_data
-                for owner, channels in owner_channel_server_data.items():
-                    for channel, svr in channels.items():
-                        old_svr = select_map_data.get(owner, {}).get(channel, [])
-                        old_svr = map(lambda x: int(x), old_svr)
-                        svr.update(old_svr)
-                        tmp_owner = select_map_data.setdefault(owner, {})
-                        tmp_owner[channel] = list(svr)
-
-                channel_alias = self.get_channel_alias()
-                server_alias = self.get_server_alias()
-
-                new_owner_channel_select_data = []
-                for owner, channels in select_map_data.items():
-                    owner_data = {
-                        'id': owner,
-                        'text': owner,
-                        'children': []
-                    }
-                    for channel, sid_list in channels.items():
-                        channel_data = {
-                            'id': channel,
-                            'text': channel_alias.get(channel, channel),
-                            'children': []
-                        }
-                        for sid in sid_list:
-                            _sid_data = {
-                                'id': sid,
-                                'text': server_alias.get(str(sid), sid),
-                            }
-                            channel_data['children'].append(_sid_data)
-
-                        owner_data['children'].append(channel_data)
-                    new_owner_channel_select_data.append(owner_data)
-
-                if select_map_data:
-                    self.local_db[self.dest_coll].update_one({'pname': 'select_map'},
-                                                             {'$set': {'data': select_map_data}}, upsert=True)
-                    self.remote_db[self.dest_coll].update_one({'pname': 'select_map'},
-                                                              {'$set': {'data': select_map_data}},
-                                                              upsert=True)
-
-                    self.local_db[self.dest_coll].update_one({'pname': 'owner_channel_select_data'},
-                                                             {'$set': {'data': new_owner_channel_select_data}},
-                                                             upsert=True)
-                    self.remote_db[self.dest_coll].update_one({'pname': 'owner_channel_select_data'},
-                                                              {'$set': {'data': new_owner_channel_select_data}},
-                                                              upsert=True)
-
-                self.set_cursor(cursor_st=ts['cursor_st'], cursor_et=ts['cursor_et'])

task/summary_func.py

@@ -35,10 +35,10 @@ class SummaryFunc(Task):
 
                 projection = self.Model.get_fields()
                 bulk_data = []
-                cdate = int(pd.Timestamp(ts['cursor_st'], unit='s', tz=self.timezone).normalize().timestamp())
                 for item in self.local_db[source_coll].find(where, projection):
                     try:
-                        item['cdate'] = cdate
+                        item['cdate'] = int(pd.Timestamp(item['_event_time'], unit='s', tz=self.timezone) \
+                                            .normalize().timestamp())
                         model = self.Model(**item)
                         data = model.dict(by_alias=True)
                         bulk_data.append(UpdateOne({'_id': data['_id']}, {'$set': data}, upsert=True))

task/summary_shopbuy.py

@@ -1,66 +0,0 @@
-from typing import List
-
-from pymongo import UpdateOne
-from pydantic import Field
-import pandas as pd
-
-from .task import Task
-from utils import *
-from model import GBaseModel, IntStr
-
-
-class SummaryShopbuy(Task):
-    """
-    功能分析
-    """
-
-    class Model(GBaseModel):
-        cdate: int = Field(..., title='当天0点')
-        prize: List[dict] = Field(None, title='奖励')
-        need: List[dict] = Field(None, title='消耗')
-        user_name: str = Field(None, title='昵称')
-        stype: IntStr = Field(None, title='商店id')
-        needa: str = Field(...)
-        needn: int = Field(...)
-        needt: str = Field(...)
-        prizea: str = Field(...)
-        prizen: int = Field(...)
-        prizet: str = Field(...)
-
-    def cleaning(self, cursor_list):
-        for cursor in cursor_list:  # type:dict
-            for source_coll, ts in cursor.items():  # type:str,dict
-                if ts['cursor_st'] == ts['cursor_et']:
-                    continue
-                logger.info(f'开始处理{self.game_name} 处理 {source_coll} 游标 {ts}')
-                where = {
-                    '_event_name': 'Shop',
-                    "act": "buy",
-                    '_event_time': {
-                        '$gte': ts['cursor_st'],
-                        '$lt': ts['cursor_et'],
-                    }
-                }
-
-                projection = self.Model.get_fields()
-                bulk_data = []
-                cdate = int(pd.Timestamp(ts['cursor_st'], unit='s', tz=self.timezone).normalize().timestamp())
-                for item in self.local_db[source_coll].find(where, projection):
-                    try:
-                        item['cdate'] = cdate
-                        item['needa'] = item['need'][0]['a']
-                        item['needt'] = item['need'][0]['t']
-                        item['needn'] = item['need'][0]['n']
-                        item['prizea'] = item['prize'][0]['a']
-                        item['prizet'] = item['prize'][0]['t']
-                        item['prizen'] = item['prize'][0]['n']
-                        model = self.Model(**item)
-                        data = model.dict(by_alias=True)
-                        bulk_data.append(UpdateOne({'_id': data['_id']}, {'$set': data}, upsert=True))
-                    except Exception as e:
-                        logger.error(f'msg:{e}')
-                        # pass
-                if bulk_data:
-                    self.local_db[self.dest_coll].bulk_write(bulk_data, ordered=False)
-                    self.remote_db[self.dest_coll].bulk_write(bulk_data, ordered=False)
-                self.set_cursor(cursor_st=ts['cursor_st'], cursor_et=ts['cursor_et'])