Compare commits

..

5 Commits

Author SHA1 Message Date
wuaho
2af12caaaf 1 2021-08-14 14:55:04 +08:00
wuaho
fc325b7518 1 2021-08-14 14:48:08 +08:00
wuaho
8a1c414008 1 2021-08-14 11:30:10 +08:00
wuaho
c6b50804d3 数据清理 2021-08-11 20:06:35 +08:00
wuaho
16ddfda32f 1 2021-08-03 17:53:00 +08:00
20 changed files with 148 additions and 273 deletions

12
Pipfile
View File

@ -4,16 +4,12 @@ verify_ssl = false
name = "pypi" name = "pypi"
[packages] [packages]
kafka-python = "2.0.2" kafka-python = "*"
clickhouse-driver = "0.2.2" clickhouse-driver = "*"
pipfile = "0.0.2" pipfile = "*"
pandas = "1.3.3" pandas = "*"
redis = "==3.5.3"
loguru = "==0.5.3"
[dev-packages] [dev-packages]
[requires] [requires]
python_version = "3.8" python_version = "3.8"

49
app.py
View File

@ -2,8 +2,6 @@
import time import time
from multiprocessing import Process from multiprocessing import Process
from kafka import TopicPartition
from settings import settings from settings import settings
from v2 import * from v2 import *
from v2.struct_cache import StructCacheFile, StructCacheRedis from v2.struct_cache import StructCacheFile, StructCacheRedis
@ -35,57 +33,38 @@ class XProcess(Process):
last_ts = int(time.time()) last_ts = int(time.time())
consumer, kafka_client = create_consumer(self.partition) consumer, kafka_client = create_consumer(self.partition)
for msg in consumer(): for topic, msg in consumer():
data = msg.value # print(msg)
type_ = data['#type'] type_ = msg['#type']
del data['#type'] del msg['#type']
ts = int(time.time()) ts = int(time.time())
try:
data['properties']['unique_id'] = f'{msg.topic}-{msg.partition}-{msg.offset}'
except:
pass
if msg.topic == 'debug':
self.log.info(data)
if 'user' in type_: if 'user' in type_:
# continue # continue
obj = getattr(handler_user, type_) obj = getattr(handler_user, type_)
handler_user.receive_data.append(UserAct(obj, data)) handler_user.receive_data.append(UserAct(obj, msg))
if len(handler_user.receive_data) >= 1000 or last_ts + 60 < ts: if len(handler_user.receive_data) >= 1000 or last_ts + 60 < ts:
last_ts = ts last_ts = ts
handler_user.execute() handler_user.execute()
elif 'track' in type_: elif 'track' in type_:
# continue # continue
if data['#event_name'] == 'pay':
self.log.info(f'topid->{msg.topic} | partition->{msg.partition} | offset->{msg.offset} | data-> {data}')
obj = getattr(handler_event, type_) obj = getattr(handler_event, type_)
obj(data) obj(msg)
elif type_ == settings.STOP_SIGNAL: elif type_ == settings.STOP_SIGNAL:
# continue
# 1 小时内有效 # 1 小时内有效
self.log.info(type_) if msg.get('#time', 0) + 3600 < int(time.time()):
if data.get('#time', 0) + 3600 < int(time.time()):
continue continue
# 停止消费kafka
self.log.info(f'进程{self.partition} 等待90秒')
time.sleep(90)
self.log.info(f'进程{self.partition} 写入数据')
transmitter.run(kafka_client)
self.log.info(f'进程{self.partition} 结束')
kafka_client.commit()
kafka_client.close() kafka_client.close()
# 停止消费kafka
print(f'进程{self.partition} 等待90秒')
time.sleep(90)
print(f'进程{self.partition} 写入数据')
transmitter.run()
print(f'进程{self.partition} 结束')
break break
elif type_ == 'test':
self.log.info(f'topid->{msg.topic} | partition->{msg.partition} | offset->{msg.offset} | data-> {data}')
else: else:
continue continue
transmitter.run(kafka_client) transmitter.run()
while True:
time.sleep(5)
self.log.info(f'消费分区{self.partition} 已结束。。。')

View File

@ -1,13 +1,8 @@
"""
清理测试数据
"""
import json import json
from settings import settings from settings import settings
game = '' game = 'zhengba'
db = settings.GAME
server_list_url = f'http://gametools.legu.cc/?app=api&act=getServerList&game={game}' server_list_url = f'http://gametools.legu.cc/?app=api&act=getServerList&game={game}'
import pandas as pd import pandas as pd
@ -20,13 +15,10 @@ df = df[~df['hostname'].isin(['119.3.89.14', '119.3.105.109'])]
serverid = tuple((str(i) for i in df['serverid'].to_list())) serverid = tuple((str(i) for i in df['serverid'].to_list()))
sql = f"""select `#account_id`, `#event_time`, `#event_name`,`#os`,`svrindex`, count() as n sql = f"""select `#account_id`, `#event_time`, `#event_name`,`#os`,`svrindex`, count() as n
from {db}.event from {game}.event
where where
`#event_time`>addDays(now('UTC'),-3) and (
lower(`#os`) = 'windows' lower(`#os`) = 'windows'
or svrindex not in {serverid} or svrindex not in {serverid}
)
group by `#account_id`, `#event_time`, `#event_name`,`#os`,`svrindex` group by `#account_id`, `#event_time`, `#event_name`,`#os`,`svrindex`
having n = 1 limit 2000""" having n = 1 limit 2000"""
@ -42,7 +34,7 @@ data_df = pd.DataFrame(
data_df.drop('n', axis=1, inplace=True) data_df.drop('n', axis=1, inplace=True)
data_df['sign'] = -1 data_df['sign'] = -1
data_df['#event_time'] = data_df['#event_time'].apply(lambda x: x.strftime('%Y-%m-%d %H:%M:%S')) data_df['#event_time'] = data_df['#event_time'].apply(lambda x: x.strftime('%Y-%m-%d %H:%M:%S'))
insert_sql = f'INSERT INTO {db}.event FORMAT JSONEachRow ' insert_sql = f'INSERT INTO {game}.event FORMAT JSONEachRow '
insert_sql = insert_sql + '\n'.join([json.dumps(item) for item in data_df.T.to_dict().values()]) insert_sql = insert_sql + '\n'.join([json.dumps(item) for item in data_df.T.to_dict().values()])
client.execute(insert_sql) client.execute(insert_sql)

View File

@ -1 +1 @@
create database xiangsu; create database zhengba;

View File

@ -5,31 +5,27 @@ class Config:
# ck数据库连接 # ck数据库连接
CK_CONFIG = {'host': '139.159.159.3', CK_CONFIG = {'host': '139.159.159.3',
'port': 9654, 'port': 9654,
'user': 'legu',
'password': 'gncPASUwpYrc'
} }
# 每个游戏不一样 游戏上报 kafka 主题 # 每个游戏不一样 游戏上报 kafka 主题
# ************* SUBSCRIBE_TOPIC = 'zhengba_test'
SUBSCRIBE_TOPIC = ''
KAFKA_CONSUMER_CONF = { KAFKA_CONSUMER_CONF = {
'bootstrap_servers': ["192.168.0.30:9092", "192.168.0.71:9092", "192.168.0.229:9092"], 'bootstrap_servers': ["192.168.0.30:9092", "192.168.0.71:9092", "192.168.0.229:9092"],
'value_deserializer': json.loads, 'value_deserializer': json.loads,
'auto_offset_reset': 'earliest', 'auto_offset_reset': 'earliest',
'enable_auto_commit': False, 'enable_auto_commit': True,
'auto_commit_interval_ms': 10000,
# 每个游戏不一样 # 每个游戏不一样
# ************* 'group_id': 'zhengba_consumer_group'
'group_id': ''
} }
KAFKA_PRODUCER_CONF = { KAFKA_PRODUCER_CONF = {
'bootstrap_servers': ["192.168.0.30:9092", "192.168.0.71:9092", "192.168.0.229:9092"], 'bootstrap_servers': ["192.168.0.30:9092", "192.168.0.71:9092", "192.168.0.229:9092"],
'value_serializer': lambda v: json.dumps(v).encode('utf-8'), 'value_serializer': lambda v: json.dumps(v).encode('utf-8'),
} }
# 游戏数据库名 # 游戏数据库名
# ************* GAME = 'zhengba'
GAME = ''
STOP_SIGNAL = 'stop_MntxuXMc' STOP_SIGNAL = 'stop_MntxuXMc'

View File

@ -1,8 +1,6 @@
# coding:utf-8
import time import time
import redis import redis
from kafka import TopicPartition
from settings import settings from settings import settings
from v2 import * from v2 import *
@ -18,29 +16,30 @@ from v2.log import logger
rdb = redis.Redis(**settings.REDIS_CONF) rdb = redis.Redis(**settings.REDIS_CONF)
event_attr = EventAttr(rdb) event_attr = EventAttr(rdb)
partition = 0
def run(): def run():
db_client = CK(**settings.CK_CONFIG) db_client = CK(**settings.CK_CONFIG)
sketch = Sketch(db_client) sketch = Sketch(db_client)
handler_event = HandlerEvent(db_client, settings.GAME, ipsearch) handler_event = HandlerEvent(db_client, settings.GAME, ipsearch)
handler_user = HandlerUser(db_client, settings.GAME) handler_user = HandlerUser(db_client, settings.GAME)
transmitter = Transmitter(db_client, settings.GAME, sketch, logger, lock, event_attr,partition) transmitter = Transmitter(db_client, settings.GAME, sketch, logger, lock, event_attr)
transmitter.add_source(handler_event, 1000, 10) transmitter.add_source(handler_event, 10000, 60)
transmitter.add_source(handler_user, 1000, 10) transmitter.add_source(handler_user, 1000, 60)
last_ts = int(time.time()) last_ts = int(time.time())
consumer, kafka_client = create_consumer(partition) consumer = create_consumer(-1)
for msg in consumer(): for topic, msg in consumer():
data = msg.value # print(msg)
type_ = data['#type'] type_ = msg['#type']
del data['#type'] if msg['#app_id']!='e903ab24ad8f4bfca8a3ce7e122cd102':
print(msg)
del msg['#type']
ts = int(time.time()) ts = int(time.time())
if 'user' in type_: if 'user' in type_:
# continue # continue
obj = getattr(handler_user, type_) obj = getattr(handler_user, type_)
handler_user.receive_data.append(UserAct(obj, data)) handler_user.receive_data.append(UserAct(obj, msg))
if len(handler_user.receive_data) >= 1000 or last_ts + 60 < ts: if len(handler_user.receive_data) >= 1000 or last_ts + 60 < ts:
last_ts = ts last_ts = ts
handler_user.execute() handler_user.execute()
@ -48,32 +47,11 @@ def run():
elif 'track' in type_: elif 'track' in type_:
# continue # continue
obj = getattr(handler_event, type_) obj = getattr(handler_event, type_)
obj(data) obj(msg)
elif type_ == settings.STOP_SIGNAL:
# continue
# 1 小时内有效
print(type_)
if data.get('#time', 0) + 3600 < int(time.time()):
continue
kafka_client.close()
# 停止消费kafka
print(f'进程{msg.partition} 等待90秒')
time.sleep(1)
print(f'进程{msg.partition} 写入数据')
transmitter.run(kafka_client)
print(f'进程{msg.partition} 结束')
break
elif type_ == 'test':
print(f'topid->{msg.topic} | partition->{msg.partition} | offset->{msg.offset} | data-> {data}')
else: else:
continue continue
transmitter.run(kafka_client) transmitter.run()
while True:
time.sleep(5)
print(f'消费分区{partition} 已结束。。。')
if __name__ == '__main__': if __name__ == '__main__':

View File

@ -1,13 +0,0 @@
import redis
from clickhouse_driver import Client
from settings import settings
rdb = redis.Redis(**settings.REDIS_CONF)
client = Client(**settings.CK_CONFIG)
sql = f"""select distinct `#event_name` as v from {settings.GAME}.event"""
df = client.query_dataframe(sql)
data = df['v'].to_list()
rdb.sadd(f'{settings.GAME}_event_set', *data)

View File

@ -1,42 +0,0 @@
# coding:utf-8
"""
更新事件表视图
"""
import json
import os
from settings import settings
game = ''
db = settings.GAME
svrid_file = f'{game}_svrid.json'
server_list_url = f'http://gametools.legu.cc/?app=api&act=getServerList&game={game}'
import pandas as pd
from clickhouse_driver import Client
client = Client(**settings.CK_CONFIG)
df = pd.read_json(server_list_url)
df = df[~df['hostname'].isin(['119.3.89.14', '119.3.105.109'])]
serverid = tuple((str(i) for i in df['serverid'].to_list()))
# if os.path.exists(svrid_file):
# with open(svrid_file, 'r') as f:
# old_svrid = json.load(f)
# if set(old_svrid) == set(serverid):
# exit(0)
sql = f"""drop table if exists {db}.event_view"""
res1 = client.execute(sql)
# 筛选有效数据
sql = f"""create view {db}.event_view as
select *
from {db}.event
where (`#os`is null or lower(`#os`) != 'windows')
and svrindex in {serverid}
and not startsWith(`orderid`,'debugPay')
"""
res2 = client.execute(sql)
with open(svrid_file, 'w') as f:
json.dump(sorted(serverid), f)

View File

@ -1,5 +1,5 @@
drop table if exists xiangsu.user_view; drop table if exists zhengba.user_view;
create view xiangsu.user_view as select * create view zhengba.user_view as select *
from xiangsu.user from zhengba.user
order by `#reg_time` desc order by `#reg_time` desc
LIMIT 1 by `#account_id` LIMIT 1 by `#account_id`

View File

@ -6,19 +6,20 @@ from settings import settings
__all__ = 'create_consumer', __all__ = 'create_consumer',
def create_consumer(partition: int = -1): def create_consumer(partition=-1):
c = KafkaConsumer(**settings.KAFKA_CONSUMER_CONF) c = KafkaConsumer(**settings.KAFKA_CONSUMER_CONF)
def consumer(): def consumer():
if partition > -1: if partition > 0:
c.assign([TopicPartition(settings.SUBSCRIBE_TOPIC, partition)]) c.assign([TopicPartition(settings.SUBSCRIBE_TOPIC, partition)])
else: else:
c.subscribe([settings.SUBSCRIBE_TOPIC]) c.subscribe([settings.SUBSCRIBE_TOPIC])
for msg in c: for msg in c:
# print(msg) # print(msg)
yield msg topic = msg.topic
# topic = msg.topic val = msg.value
# val = msg.value if val.get('properties',{}).get('owner_name') not in ('gmhdgdt', 'gmhdtt'):
# yield topic, val continue
yield topic, val
return consumer, c return consumer, c

View File

@ -1,6 +1,5 @@
__all__ = 'CK', __all__ = 'CK',
import threading
import traceback import traceback
import pandas as pd import pandas as pd
@ -16,20 +15,12 @@ class CK:
self.args = args self.args = args
self.kwargs = kwargs self.kwargs = kwargs
self.__client = self.__create_client() self.__client = self.__create_client()
self.look = threading.Lock()
def __create_client(self): def __create_client(self):
return Client(*self.args, **self.kwargs) return Client(*self.args, **self.kwargs)
def execute(self, *args, **kwargs): def execute(self, *args, **kwargs):
try: return self.__client.execute(*args, **kwargs)
self.look.acquire(timeout=10)
res = self.__client.execute(*args, **kwargs)
except Exception as e:
raise e
finally:
self.look.release()
return res
def get_one(self, db, tb, try_cnt=3, **where): def get_one(self, db, tb, try_cnt=3, **where):
@ -39,7 +30,7 @@ class CK:
sql += ' limit 1' sql += ' limit 1'
data = None data = None
try: try:
data, columns = self.execute(sql, with_column_types=True) data, columns = self.__client.execute(sql, with_column_types=True)
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
self.__client.disconnect() self.__client.disconnect()
@ -71,9 +62,9 @@ class CK:
sql += where sql += where
data = None data = None
try: try:
data, columns = self.execute(sql, columnar=True, with_column_types=True) data, columns = self.__client.execute(sql, columnar=True, with_column_types=True)
except Exception as e: except Exception as e:
print('*' * 50) print('*'*50)
print(sql) print(sql)
traceback.print_exc() traceback.print_exc()
if e.code == 60: if e.code == 60:

View File

@ -7,12 +7,26 @@ class EventAttr:
def __init__(self, rdb: Redis): def __init__(self, rdb: Redis):
self.rdb = rdb self.rdb = rdb
def set_event_name(self, key, *data): def get_event_attr(self, key):
self.rdb.sadd(key, *data) attr = self.event_attr.get(key)
if not attr:
self.event_attr[key] = self.rdb.smembers(key) or set()
return set(self.event_attr[key])
def add_event(self, db, data): def set_event_attr(self, key, *data):
self.rdb.sadd(key, *data)
self.event_attr[key] = data
def check_attr(self, db, data):
event_name = data.get('#event_name') event_name = data.get('#event_name')
if not event_name: if not event_name:
return return
key = f'{db}_event_set'
self.set_event_name(key, event_name) key = f'{db}_event_{event_name}'
attr = self.get_event_attr(key)
data_attr = set(data)
extra_attr = data_attr - attr
if extra_attr:
self.set_event_attr(key, *extra_attr)

View File

@ -98,7 +98,7 @@ class Sketch:
default_field[k] = 'Nullable(String)' default_field[k] = 'Nullable(String)'
if isinstance(v, int): if isinstance(v, int):
default_field[k] = 'Nullable(Int64)' default_field[k] = 'Nullable(UInt64)'
if isinstance(v, float): if isinstance(v, float):
default_field[k] = 'Nullable(Float32)' default_field[k] = 'Nullable(Float32)'
@ -109,9 +109,9 @@ class Sketch:
if isinstance(v, bool): if isinstance(v, bool):
default_field[k] = 'Nullable(UInt8)' default_field[k] = 'Nullable(UInt8)'
sql = f'alter table {db}.{tb} add column `{k}` {default_field[k]}'
print(sql)
try: try:
sql = f'alter table {db}.{tb} add column `{k}` {default_field[k]}'
print(sql)
self.db_client.execute(sql) self.db_client.execute(sql)
except Exception as e: except Exception as e:
print(f'添加字段 {k} 失败,同步数据库表结构') print(f'添加字段 {k} 失败,同步数据库表结构')

View File

@ -5,8 +5,6 @@ import threading
import time import time
import traceback import traceback
from settings import settings
from .valid_data import * from .valid_data import *
__all__ = 'Transmitter', __all__ = 'Transmitter',
@ -24,7 +22,6 @@ class Ping(threading.Thread):
def run(self): def run(self):
while True: while True:
time.sleep(10)
ts = int(time.time()) ts = int(time.time())
if self.ping_ts + self.time_out < ts: if self.ping_ts + self.time_out < ts:
# 保持连接 # 保持连接
@ -82,11 +79,9 @@ class Transmitter:
self.db_client.execute(sql) self.db_client.execute(sql)
self.log.info(f'进程{self.p} 写入耗时 {int(time.time() * 1000) - ts}') self.log.info(f'进程{self.p} 写入耗时 {int(time.time() * 1000) - ts}')
except Exception as e: except Exception as e:
self.log.error(traceback.format_exc())
# 丢弃错误行 再次发送 # 丢弃错误行 再次发送
if hasattr(e, 'code') and e.code == 26: if hasattr(e, 'code') and e.code == 26:
m = re.match('(.*)?Stack trace', e.message) m = re.match('(.*)?Stack trace', e.message)
self.log.error(data)
if m: if m:
error_msg = m.group(1) error_msg = m.group(1)
error_row = re.match('.*?errors out of (\d+) rows', error_msg) error_row = re.match('.*?errors out of (\d+) rows', error_msg)
@ -108,11 +103,10 @@ class Transmitter:
def check_table(self, db, tb, data): def check_table(self, db, tb, data):
[self.sketch.alter_table(db, tb, item) for item in data] [self.sketch.alter_table(db, tb, item) for item in data]
def collect_event(self, db, tb, data): def set_event_attr(self, db, tb, data):
if tb != 'event': if tb != 'event':
return return
[self.event_attr.check_attr(db, item) for item in data]
[self.event_attr.add_event(db, item) for item in data]
def check_type(self, db, tb, data): def check_type(self, db, tb, data):
struct_dict = self.sketch.struct_dict[f'{db}_{tb}'] struct_dict = self.sketch.struct_dict[f'{db}_{tb}']
@ -130,19 +124,14 @@ class Transmitter:
for key in del_keys: for key in del_keys:
del item[key] del item[key]
def run(self, kafka_client): def run(self):
for tb, buffer in self.check_send(): for tb, buffer in self.check_send():
try: try:
data = [self.flat_data(x) for x in buffer.values()] data = [self.flat_data(x) for x in buffer.values()]
self.check_table(self.db_name, tb, data) self.check_table(self.db_name, tb, data)
self.check_type(self.db_name, tb, data) self.check_type(self.db_name, tb, data)
self.collect_event(self.db_name, tb, data) self.set_event_attr(self.db_name, tb, data)
self.__send(self.db_name, tb, [json.dumps(item) for item in data]) self.__send(self.db_name, tb, [json.dumps(item) for item in data])
except Exception as e: except Exception as e:
self.log.error(traceback.format_exc()) self.log.error(e)
self.log.error(data)
buffer.clear() buffer.clear()
try:
kafka_client.commit()
except Exception as e:
self.log.error(f'进程:{self.p} error:{e}')

View File

@ -1,5 +1,5 @@
drop table if exists xiangsu.recharge_game; drop table if exists zhengba.recharge_game;
create view xiangsu.recharge_game as create view zhengba.recharge_game as
select toDate(addHours(`#event_time`, `#zone_offset`)) as date, select toDate(addHours(`#event_time`, `#zone_offset`)) as date,
`#os`, `#os`,
`#bundle_id`, `#bundle_id`,
@ -8,6 +8,6 @@ select toDate(addHours(`#event_time`, `#zone_offset`)) as date,
arrayDistinct(groupArray(binduid)) as account, arrayDistinct(groupArray(binduid)) as account,
length(account) as account_num, length(account) as account_num,
sum(money) as money sum(money) as money
from xiangsu.event from zhengba.event
where `#event_name` = 'rechargeGame' where `#event_name` = 'rechargeGame'
group by toDate(addHours(`#event_time`, `#zone_offset`)), `#os`, `#bundle_id`, owner_name, channel group by toDate(addHours(`#event_time`, `#zone_offset`)), `#os`, `#bundle_id`, owner_name, channel

View File

@ -1,61 +1,55 @@
-- auto-generated definition drop table if exists zhengba.event;
create table event create table zhengba.event
( (
`#ip` Nullable(IPv4), `#ip` Nullable(IPv4),
`#country` Nullable(String), `#country` Nullable(String),
`#province` Nullable(String), `#country_code` Nullable(String),
`#city` Nullable(String), `#province` Nullable(String),
`#os` Nullable(String), `#city` Nullable(String),
`#device_id` Nullable(String), `#os_version` Nullable(String),
`#screen_height` Nullable(UInt16), `#manufacturer` Nullable(String),
`#screen_width` Nullable(UInt16), `#os` Nullable(String),
`#device_model` Nullable(String), `#device_id` Nullable(String),
`#app_version` Nullable(String), `#screen_height` Nullable(UInt16),
`#bundle_id` Nullable(String), `#screen_width` Nullable(UInt16),
`#app_name` Nullable(String), `#device_model` Nullable(String),
`#game_version` Nullable(String), `#app_version` Nullable(String),
`#os_version` Nullable(String), `#bundle_id` Nullable(String),
`#network_type` Nullable(String), `#lib` Nullable(String),
`#carrier` Nullable(String), `#lib_version` Nullable(String),
`#manufacturer` Nullable(String), `#network_type` Nullable(String),
`#app_id` Nullable(String), `#carrier` Nullable(String),
`#account_id` String, `#browser` Nullable(String),
`#distinct_id` Nullable(String), `#browser_version` Nullable(String),
binduid Nullable(String), `#duration` Nullable(String),
channel Nullable(String), `#url` Nullable(String),
owner_name String default '', `#url_path` Nullable(String),
role_name Nullable(String), `#referrer` Nullable(String),
exp Nullable(UInt64), `#referrer_host` Nullable(String),
zhanli Nullable(UInt64), `#title` Nullable(String),
maxmapid Nullable(UInt16), `#screen_name` Nullable(String),
mapid Nullable(UInt16), `#element_id` Nullable(String),
ghid Nullable(String), `#element_type` Nullable(String),
rmbmoney Nullable(UInt64), `#resume_from_background` Nullable(String),
jinbi Nullable(UInt64), `#element_selector` Nullable(String),
svrindex Nullable(String), `#element_position` Nullable(String),
lv Nullable(UInt16), `#element_content` Nullable(String),
vip Nullable(UInt16), `#scene` Nullable(String),
game Nullable(String), `#mp_platform` Nullable(String),
`#app_crashed_reason` Nullable(String),
`#zone_offset` Int8 default 8,
`#event_id` String,
`#zone_offset` Int8 default 8, `#event_time` DateTime('UTC'),
`#event_time` DateTime('UTC'), `#account_id` String,
`#event_name` String, `#distinct_id` Nullable(String),
`#server_time` DateTime('UTC') default now(), `#event_name` String,
`#server_time` DateTime('UTC') default now(),
unitPrice Nullable(UInt32),
money Nullable(String),
islishishouci Nullable(UInt8),
isdangrishouci Nullable(UInt8),
is_today_reg Nullable(UInt8),
orderid Nullable(String),
proid Nullable(String),
step_id Nullable(UInt16),
step_group Nullable(UInt16),
guide_start_time Nullable(UInt32),
online_ts Nullable(UInt16),
`#time` Nullable(DateTime('UTC'))
)
engine = ReplacingMergeTree PARTITION BY toYYYYMMDD(`#event_time`)
ORDER BY (owner_name, `#event_name`, `#event_time`, `#account_id`)
SETTINGS index_granularity = 8192;
`sign` Int8 default 1
) ENGINE = CollapsingMergeTree(sign)
PARTITION BY toYYYYMMDD(`#event_time`)
order by (`#account_id`, `#event_time`, `#event_name`)
-- TTL event_time + toIntervalDay(365)

View File

@ -1,10 +1,10 @@
drop table if exists xiangsu.user; drop table if exists zhengba.user;
create table xiangsu.user create table zhengba.user
( (
`#reg_time` DateTime('UTC'), `#reg_time` DateTime('UTC'),
`#account_id` String, `#account_id` String,
`svrindex` String, `svrindex` UInt16,
`#zone_offset` Int8 default 8, `#zone_offset` Int8 default 8,
`#server_time` DateTime('UTC') default now() `#server_time` DateTime('UTC') default now()

View File

@ -1,5 +1,5 @@
drop table if exists xiangsu.new_account; drop table if exists zhengba.new_account;
create view xiangsu.new_account as create view zhengba.new_account as
select toDate(addHours(`#event_time`, `#zone_offset`)) as date, select toDate(addHours(`#event_time`, `#zone_offset`)) as date,
`#os`, `#os`,
`#bundle_id`, `#bundle_id`,
@ -7,6 +7,6 @@ select toDate(addHours(`#event_time`, `#zone_offset`)) as date,
channel, channel,
groupArray(`binduid`) as account, groupArray(`binduid`) as account,
length(account) as num length(account) as num
from xiangsu.event from zhengba.event
where role_idx = 1 where role_idx = 1
group by toDate(addHours(`#event_time`, `#zone_offset`)), `#os`, `#bundle_id`, owner_name, channel group by toDate(addHours(`#event_time`, `#zone_offset`)), `#os`, `#bundle_id`, owner_name, channel

View File

@ -1,5 +1,5 @@
drop table if exists xiangsu.new_account_recharge; drop table if exists zhengba.new_account_recharge;
create view xiangsu.new_account_recharge as (select date, create view zhengba.new_account_recharge as (select date,
`#os`, `#os`,
`#bundle_id`, `#bundle_id`,
owner_name, owner_name,
@ -13,11 +13,11 @@ from (select toDate(addHours(`#event_time`, `#zone_offset`)) as date,
channel, channel,
binduid, binduid,
money money
from xiangsu.event from zhengba.event
where `#event_name` = 'rechargeGame') as tb1 where `#event_name` = 'rechargeGame') as tb1
right join (select toDate(addHours(`#event_time`, `#zone_offset`)) as date, right join (select toDate(addHours(`#event_time`, `#zone_offset`)) as date,
binduid binduid
from xiangsu.event from zhengba.event
where role_idx = 1) as tb2 where role_idx = 1) as tb2
on tb1.date = tb2.date and tb2.binduid = tb1.binduid on tb1.date = tb2.date and tb2.binduid = tb1.binduid
group by date, `#os`, `#bundle_id`, owner_name, channel) group by date, `#os`, `#bundle_id`, owner_name, channel)

View File

@ -1,5 +1,5 @@
drop table if exists xiangsu.active_account; drop table if exists zhengba.active_account;
create view xiangsu.active_account as create view zhengba.active_account as
select toDate(addHours(`#event_time`, `#zone_offset`)) as date, select toDate(addHours(`#event_time`, `#zone_offset`)) as date,
`#os`, `#os`,
`#bundle_id`, `#bundle_id`,
@ -7,5 +7,5 @@ select toDate(addHours(`#event_time`, `#zone_offset`)) as date,
channel, channel,
uniqCombined(binduid) as num, uniqCombined(binduid) as num,
arrayDistinct(groupArray(binduid)) as account arrayDistinct(groupArray(binduid)) as account
from xiangsu.event from zhengba.event
group by toDate(addHours(`#event_time`, `#zone_offset`)), `#os`, `#bundle_id`, owner_name, channel group by toDate(addHours(`#event_time`, `#zone_offset`)), `#os`, `#bundle_id`, owner_name, channel