data_cleaning/main.py
2020-12-11 10:51:07 +08:00

38 lines
978 B
Python

import sys
from multiprocessing import Pool
from importlib import import_module
from db import *
def get_game() -> list:
local_db = get_local_db('admin_game')
games = list(local_db['game'].find())
return games
def run_task(kwargs):
module_name = kwargs.get('task_name')
class_name = ''.join([s.capitalize() for s in task_name.split('_')])
module = import_module(f'.{module_name}', package='task')
c_obj = getattr(module, class_name)
obj = c_obj(**kwargs)
obj.run()
if __name__ == '__main__':
# eg: summary_func 0 0
task_name, st, et = sys.argv[1:]
st, et = int(st), int(et)
game_list = get_game()
params = [{'game_name': item['id_name'],
'task_name': task_name,
'timezone': item.get('timezone', 'Asia/Shanghai'),
'st': st,
'et': et
}
for item in game_list]
with Pool(len(game_list)) as p:
p.map(run_task, params)