wuhao/data_cleaning
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

update

Browse Source
This commit is contained in:
kf_wuhao 2020-12-23 17:25:27 +08:00
parent 20067fd558
commit cf2d7d5703
5 changed files with 76 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
config.json
View File

@ -9,9 +9,15 @@
"dest_coll": "user",
"task_name": "first_recharge"
},
"repair_gunfu": {
"repair_gunfu": {
"source_coll": "user",
"dest_coll": "user",
"task_name": "repair_gunfu"
},
"summary_login": {
"source_coll": "event",
"dest_coll": "summary_login",
"task_name": "summary_login"
}
}

2
task/first_recharge.py
View File

@ -55,6 +55,6 @@ class FirstRecharge(Task):
logger.error(f'msg:{e}')
# pass
if bulk_data:
self.remote_db[self.dest_coll].bulk_write(bulk_data, ordered=False)
self.local_db[self.dest_coll].bulk_write(bulk_data, ordered=False)
self.remote_db[self.dest_coll].bulk_write(bulk_data, ordered=False)
self.set_cursor(cursor_st=ts['cursor_st'], cursor_et=ts['cursor_et'])

7
task/repair_gunfu.py
View File

@ -4,7 +4,6 @@ from pydantic import BaseModel, Field
from .task import Task
from utils import *
from model import MdbObjectId
class RepairGunfu(Task):
@ -13,7 +12,7 @@ class RepairGunfu(Task):
"""
class Model(BaseModel):
id: MdbObjectId = Field(..., title='_id', alias='_id')
game_role_id: str = Field(None, title="角色id", alias='_game_role_id')
device_id: str = Field(..., title='设备id', alias='_device_id')
@classmethod
@ -47,12 +46,12 @@ class RepairGunfu(Task):
{'_device_id': device_id, 'gunfu_num': {'$exists': True}})
bulk_data.append(
UpdateOne({'_id': model.id},
UpdateOne({'_game_role_id': model.game_role_id},
{'$set': {'gunfu_num': role_cnt + 1}}, upsert=True))
except Exception as e:
logger.error(f'msg:{e}')
# pass
if bulk_data:
self.remote_db[self.dest_coll].bulk_write(bulk_data, ordered=False)
self.local_db[self.dest_coll].bulk_write(bulk_data, ordered=False)
self.remote_db[self.dest_coll].bulk_write(bulk_data, ordered=False)
self.set_cursor(cursor_st=ts['cursor_st'], cursor_et=ts['cursor_et'])

1
task/summary_func.py
View File

@ -46,5 +46,6 @@ class SummaryFunc(Task):
logger.error(f'ftype {item["ftype"]} msg:{e}')
# pass
if bulk_data:
self.local_db[self.dest_coll].bulk_write(bulk_data, ordered=False)
self.remote_db[self.dest_coll].bulk_write(bulk_data, ordered=False)
self.set_cursor(cursor_st=ts['cursor_st'], cursor_et=ts['cursor_et'])

64
task/summary_login.py Normal file
View File

@ -0,0 +1,64 @@
from pymongo import UpdateOne
from pydantic import Field
import pandas as pd
from .task import Task
from utils import *
from model import GBaseModel
class SummaryLogin(Task):
"""
每日活跃
"""
class Model(GBaseModel):
cdate: int = Field(..., title='当天0点')
# first_device_id: str = Field(None, title='第一次登录设备id', alias='_first_device_id')
manufacturer: str = Field(None, title='设备品牌', alias='_manufacturer')
model: str = Field(None, title='型号', alias='_model')
os_version: str = Field(None, title='系统版本', alias='_os_version')
app_name: str = Field(None, title='游戏版本', alias='_app_name')
screen_width: int = Field(None, title='屏幕宽', alias='_screen_width')
screen_height: int = Field(None, title='屏幕高', alias='_screen_height')
def cleaning(self, cursor_list):
for cursor in cursor_list: # type:dict
for source_coll, ts in cursor.items(): # type:str,dict
if ts['cursor_st'] == ts['cursor_et']:
continue
logger.info(f'开始处理{self.game_name} 处理 {source_coll} 游标 {ts}')
where = {
'_event_time': {
'$gte': ts['cursor_st'],
'$lt': ts['cursor_et'],
},
'_event_name': 'TimeSpending' # 在线时长打点 30s
}
projection = self.Model.get_fields()
bulk_data = []
# 当前活跃role id
role_list = self.local_db[source_coll].distinct('_game_role_id', where)
# 当日已记录的活跃role
cdate = int(pd.Timestamp(ts['cursor_st'], unit='s', tz=self.timezone) \
.normalize().timestamp())
exists_role_list = self.local_db[self.dest_coll].distinct('_game_role_id', {'cdate': cdate})
# 还没有记录的
role_set = set(role_list) - set(exists_role_list)
for role_id in role_set:
item = self.local_db[source_coll].find_one(
{'_game_role_id': role_id, '_event_name': 'TimeSpending'}, projection)
try:
item['cdate'] = cdate
model = self.Model(**item)
data = model.dict(by_alias=True)
bulk_data.append(UpdateOne({'_id': data['_id']}, {'$set': data}, upsert=True))
except Exception as e:
logger.error(f'msg:{e}')
# pass
if bulk_data:
self.local_db[self.dest_coll].bulk_write(bulk_data, ordered=False)
self.remote_db[self.dest_coll].bulk_write(bulk_data, ordered=False)
self.set_cursor(cursor_st=ts['cursor_st'], cursor_et=ts['cursor_et'])