leguadmin/to_ck
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

改为手动提交

Browse Source
This commit is contained in:
wuaho 2021-10-26 19:55:54 +08:00
parent cf46c8b6d0
commit dad6c0b072
1 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
v2/transmitter.py
View File

@ -5,6 +5,9 @@ import threading
import time import time
import traceback import traceback
from kafka import TopicPartition, OffsetAndMetadata
from settings import settings
from .valid_data import * from .valid_data import *
__all__ = 'Transmitter', __all__ = 'Transmitter',
@ -44,6 +47,7 @@ class Transmitter:
self.lock = lock self.lock = lock
self.event_attr = event_attr self.event_attr = event_attr
self.p = p self.p = p
self.topic_p = TopicPartition(settings.SUBSCRIBE_TOPIC, p)
def start_ping(self): def start_ping(self):
t = Ping(self.db_client, self.p, self.log) t = Ping(self.db_client, self.p, self.log)
@ -106,8 +110,6 @@ class Transmitter:
def check_table(self, db, tb, data): def check_table(self, db, tb, data):
[self.sketch.alter_table(db, tb, item) for item in data] [self.sketch.alter_table(db, tb, item) for item in data]
def collect_event(self, db, tb, data): def collect_event(self, db, tb, data):
if tb != 'event': if tb != 'event':
return return
@ -141,7 +143,8 @@ class Transmitter:
except Exception as e: except Exception as e:
self.log.error(traceback.format_exc()) self.log.error(traceback.format_exc())
buffer.clear() buffer.clear()
# try: try:
# kafka_client.commit() offsets = kafka_client.committed(self.topic_p)
# except Exception as e: kafka_client.commit({self.topic_p: OffsetAndMetadata(offsets, '')})
# self.log.error(f'进程:{self.p} error:{e}') except Exception as e:
self.log.error(f'进程:{self.p} error:{e}')