From 4877426a45200231025d347714be57df0c55a900 Mon Sep 17 00:00:00 2001 From: kf_wuhao <15392746632@qq.com> Date: Mon, 18 Jan 2021 19:54:41 +0800 Subject: [PATCH] update --- model/__init__.py | 2 +- model/model.py | 4 ++-- task/add_user_flag.py | 36 +++++++++++++++++++++++++++--------- task/first_recharge.py | 12 +++++++----- task/summary_login.py | 12 ++++++++++-- task/summary_pay.py | 14 ++++++++++---- 6 files changed, 57 insertions(+), 23 deletions(-) diff --git a/model/__init__.py b/model/__init__.py index 0d955e3..d902d01 100644 --- a/model/__init__.py +++ b/model/__init__.py @@ -1,2 +1,2 @@ from .field_type import (IntStr, IntFloat, MdbObjectId) -from .model import (GBaseModel, BaseModel) +from .model import (GBaseModel, BaseModel, ValidationError) diff --git a/model/model.py b/model/model.py index 3f45d29..200521f 100644 --- a/model/model.py +++ b/model/model.py @@ -2,7 +2,7 @@ from pydantic import BaseModel as BModel from pydantic import Field from model.field_type import IntStr - +from pydantic.error_wrappers import ValidationError class BaseModel(BModel): @classmethod @@ -11,7 +11,7 @@ class BaseModel(BModel): class GBaseModel(BaseModel): - platform: str = Field(None, title="平台", alias='_platform') + platform: str = Field(..., min_length=1, title="平台", alias='_platform') channel_name: str = Field(..., min_length=1, title="channel", alias='_channel_name') owner_name: str = Field(..., min_length=1, title="owner", alias='_owner_name') channel_uid: str = Field(..., min_length=1, title="channel_uid", alias='_channel_uid') diff --git a/task/add_user_flag.py b/task/add_user_flag.py index 1c3b519..23f25b3 100644 --- a/task/add_user_flag.py +++ b/task/add_user_flag.py @@ -4,7 +4,7 @@ import pymongo from pymongo import UpdateOne from pydantic import Field -from model import BaseModel +from model import BaseModel, ValidationError from .task import Task from utils import * @@ -15,9 +15,9 @@ class AddUserFlag(Task): """ class Model(BaseModel): - game_role_id: str = Field(..., min_length=1, title="角色id", alias='_game_role_id') - device_id: str = Field(..., min_length=1, title='设备id', alias='_device_id') - channel_uid: str = Field(..., min_length=1, title="channel_uid", alias='_channel_uid') + game_role_id: str = Field(..., min_length=5, title="角色id", alias='_game_role_id') + device_id: str = Field(..., min_length=5, title='设备id', alias='_device_id') + channel_uid: str = Field(..., min_length=5, title="channel_uid", alias='_channel_uid') role_create_time: int = Field(..., title="注册时间戳") def cleaning(self, cursor_list): @@ -49,9 +49,15 @@ class AddUserFlag(Task): bulk_data.append( UpdateOne({'_game_role_id': model.game_role_id}, {'$set': {'is_new_channel_uid': 1}})) + except ValidationError as e: + msg = traceback.format_exc() + ddsend_msg(f'{self.game_name}.{source_coll}字段验证异常 {msg}\n{e.json()}') + logger.error(repr(e)) + except Exception as e: - logger.error(f'msg:{e}') - # pass + msg = traceback.format_exc() + ddsend_msg(f'未知异常 {msg}') + logger.error(repr(e)) # 处理设备 @@ -74,9 +80,15 @@ class AddUserFlag(Task): bulk_data.append( UpdateOne({'_game_role_id': model.game_role_id}, {'$set': {'is_new_device': 1}})) + except ValidationError as e: + msg = traceback.format_exc() + ddsend_msg(f'{self.game_name}.{source_coll}字段验证异常 {msg}\n{e.json()}') + logger.error(repr(e)) + except Exception as e: - logger.error(f'msg:{e}') - pass + msg = traceback.format_exc() + ddsend_msg(f'未知异常 {msg}') + logger.error(repr(e)) # 记录第一次登录设备id where = { @@ -92,9 +104,15 @@ class AddUserFlag(Task): model = self.Model(**item) bulk_data.append( UpdateOne({'_game_role_id': model.game_role_id}, {'$set': {'_first_device_id': model.device_id}})) + + except ValidationError as e: + msg = traceback.format_exc() + ddsend_msg(f'{self.game_name}.{source_coll}字段验证异常 {msg}\n{e.json()}') + logger.error(repr(e)) + except Exception as e: msg = traceback.format_exc() - ddsend_msg(f'{self.game_name}.{source_coll}字段异常 {msg}') + ddsend_msg(f'未知异常 {msg}') logger.error(repr(e)) if bulk_data: diff --git a/task/first_recharge.py b/task/first_recharge.py index 4422d82..e5a0513 100644 --- a/task/first_recharge.py +++ b/task/first_recharge.py @@ -6,7 +6,7 @@ import pandas as pd from .task import Task from utils import * -from model import IntStr, IntFloat, BaseModel +from model import IntStr, IntFloat, BaseModel, ValidationError class FirstRecharge(Task): @@ -25,9 +25,6 @@ class FirstRecharge(Task): proid: str = Field(..., min_length=1, title='计费点') cdate: int = Field(..., title='当天0点') - @classmethod - def get_fields(cls): - return [v.alias for v in cls.__fields__.values()] def cleaning(self, cursor_list): for cursor in cursor_list: # type:dict @@ -54,9 +51,14 @@ class FirstRecharge(Task): bulk_data.append( UpdateOne({'_game_role_id': _game_role_id, 'is_recharge': {'$exists': False}}, {'$set': {'is_recharge': data}})) + except ValidationError as e: + msg = traceback.format_exc() + ddsend_msg(f'{self.game_name}.{source_coll}字段验证异常 {msg}\n{e.json()}') + logger.error(repr(e)) + except Exception as e: msg = traceback.format_exc() - ddsend_msg(f'{self.game_name}.{source_coll}字段异常 {msg}') + ddsend_msg(f'未知异常 {msg}') logger.error(repr(e)) if bulk_data: self.local_db[self.dest_coll].bulk_write(bulk_data, ordered=False) diff --git a/task/summary_login.py b/task/summary_login.py index 27f9745..861e532 100644 --- a/task/summary_login.py +++ b/task/summary_login.py @@ -6,7 +6,7 @@ import pandas as pd from .task import Task from utils import * -from model import GBaseModel +from model import GBaseModel, ValidationError class SummaryLogin(Task): @@ -56,9 +56,17 @@ class SummaryLogin(Task): bulk_data.append( UpdateOne({'cdate': cdate, '_game_role_id': data['_game_role_id']}, {'$set': data}, upsert=True)) + except ValidationError as e: + bulk_data.append( + UpdateOne({'cdate': cdate, '_game_role_id': item['_game_role_id']}, {'$set': item}, + upsert=True)) + msg = traceback.format_exc() + ddsend_msg(f'{self.game_name}.{source_coll}字段验证异常 {msg}\n{e.json()}') + logger.error(repr(e)) + except Exception as e: msg = traceback.format_exc() - ddsend_msg(f'{self.game_name}.{source_coll}字段异常 {msg}') + ddsend_msg(f'未知异常 {msg}') logger.error(repr(e)) if bulk_data: self.local_db[self.dest_coll].bulk_write(bulk_data, ordered=False) diff --git a/task/summary_pay.py b/task/summary_pay.py index 0b1d58a..76572bd 100644 --- a/task/summary_pay.py +++ b/task/summary_pay.py @@ -6,7 +6,7 @@ import pandas as pd from .task import Task from utils import * -from model import IntFloat, GBaseModel +from model import IntFloat, GBaseModel, ValidationError class SummaryPay(Task): @@ -53,7 +53,7 @@ class SummaryPay(Task): orderid.startswith('debugPay'): continue item['cdate'] = int(pd.Timestamp(item['_event_time'], unit='s', tz=self.timezone) \ - .normalize().timestamp()) + .normalize().timestamp()) user_info = self.local_db['user'].find_one({'_game_role_id': item['_game_role_id']}, projection) for k, v in user_info.items(): item[k] = item.get(k) or user_info[k] @@ -61,11 +61,17 @@ class SummaryPay(Task): model = self.Model(**item) data = model.dict(by_alias=True) bulk_data.append(UpdateOne({'orderid': model.orderid}, {'$set': data}, upsert=True)) + except ValidationError as e: + bulk_data.append(UpdateOne({'orderid': item['orderid']}, {'$set': item}, upsert=True)) + msg = traceback.format_exc() + ddsend_msg(f'{self.game_name}.{source_coll}字段验证异常 {msg}\n{e.json()}') + logger.error(repr(e)) + except Exception as e: msg = traceback.format_exc() - ddsend_msg(f'严重警告！！！{self.game_name}.{source_coll}字段异常 {msg}') - bulk_data.append(UpdateOne({'orderid': item['orderid']}, {'$set': item}, upsert=True)) + ddsend_msg(f'未知异常 {msg}') logger.error(repr(e)) + if bulk_data: self.local_db[self.dest_coll].bulk_write(bulk_data, ordered=False) self.remote_db[self.dest_coll].bulk_write(bulk_data, ordered=False)