增加Redis查询比对

This commit is contained in:
2025-08-04 09:14:27 +08:00
parent e1a566012d
commit 4c4d168471
18 changed files with 8007 additions and 2250 deletions

671
modules/config_manager.py Normal file
View File

@@ -0,0 +1,671 @@
"""
配置管理模块
负责配置组和查询历史的CRUD操作
"""
import json
import logging
from datetime import datetime
from .database import ensure_database, get_db_connection
logger = logging.getLogger(__name__)
# 默认配置(不显示敏感信息)
DEFAULT_CONFIG = {
'pro_config': {
'cluster_name': '',
'hosts': [],
'port': 9042,
'datacenter': '',
'username': '',
'password': '',
'keyspace': '',
'table': ''
},
'test_config': {
'cluster_name': '',
'hosts': [],
'port': 9042,
'datacenter': '',
'username': '',
'password': '',
'keyspace': '',
'table': ''
},
'keys': [],
'fields_to_compare': [],
'exclude_fields': []
}
# Redis默认配置
REDIS_DEFAULT_CONFIG = {
'cluster1_config': {
'name': '生产集群',
'nodes': [
{'host': '127.0.0.1', 'port': 7000}
],
'password': '',
'socket_timeout': 3,
'socket_connect_timeout': 3,
'max_connections_per_node': 16
},
'cluster2_config': {
'name': '测试集群',
'nodes': [
{'host': '127.0.0.1', 'port': 7001}
],
'password': '',
'socket_timeout': 3,
'socket_connect_timeout': 3,
'max_connections_per_node': 16
},
'query_options': {
'mode': 'random',
'count': 100,
'pattern': '*',
'source_cluster': 'cluster2',
'keys': []
}
}
def save_redis_config_group(name, description, cluster1_config, cluster2_config, query_options):
"""保存Redis配置组"""
if not ensure_database():
logger.error("数据库初始化失败")
return False
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('''
INSERT OR REPLACE INTO redis_config_groups
(name, description, cluster1_config, cluster2_config, query_options, updated_at)
VALUES (?, ?, ?, ?, ?, ?)
''', (
name, description,
json.dumps(cluster1_config),
json.dumps(cluster2_config),
json.dumps(query_options),
datetime.now().isoformat()
))
conn.commit()
logger.info(f"Redis配置组 '{name}' 保存成功")
return True
except Exception as e:
logger.error(f"保存Redis配置组失败: {e}")
return False
finally:
conn.close()
def get_redis_config_groups():
"""获取所有Redis配置组"""
if not ensure_database():
logger.error("数据库初始化失败")
return []
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('''
SELECT id, name, description, created_at, updated_at
FROM redis_config_groups
ORDER BY updated_at DESC
''')
rows = cursor.fetchall()
config_groups = []
for row in rows:
config_groups.append({
'id': row['id'],
'name': row['name'],
'description': row['description'],
'created_at': row['created_at'],
'updated_at': row['updated_at']
})
return config_groups
except Exception as e:
logger.error(f"获取Redis配置组失败: {e}")
return []
finally:
conn.close()
def get_redis_config_group_by_id(group_id):
"""根据ID获取Redis配置组详情"""
if not ensure_database():
logger.error("数据库初始化失败")
return None
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('''
SELECT id, name, description, cluster1_config, cluster2_config, query_options,
created_at, updated_at
FROM redis_config_groups WHERE id = ?
''', (group_id,))
row = cursor.fetchone()
if row:
config = {
'id': row['id'],
'name': row['name'],
'description': row['description'],
'cluster1_config': json.loads(row['cluster1_config']),
'cluster2_config': json.loads(row['cluster2_config']),
'query_options': json.loads(row['query_options']),
'created_at': row['created_at'],
'updated_at': row['updated_at']
}
return config
return None
except Exception as e:
logger.error(f"获取Redis配置组详情失败: {e}")
return None
finally:
conn.close()
def delete_redis_config_group(group_id):
"""删除Redis配置组"""
if not ensure_database():
logger.error("数据库初始化失败")
return False
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('DELETE FROM redis_config_groups WHERE id = ?', (group_id,))
conn.commit()
success = cursor.rowcount > 0
if success:
logger.info(f"Redis配置组ID {group_id} 删除成功")
return success
except Exception as e:
logger.error(f"删除Redis配置组失败: {e}")
return False
finally:
conn.close()
def save_redis_query_history(name, description, cluster1_config, cluster2_config, query_options,
query_keys, results_summary, execution_time, total_keys,
different_count, identical_count, missing_count, raw_results=None):
"""保存Redis查询历史记录返回历史记录ID"""
if not ensure_database():
logger.error("数据库初始化失败")
return None
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('''
INSERT INTO redis_query_history
(name, description, cluster1_config, cluster2_config, query_options, query_keys,
results_summary, execution_time, total_keys, different_count, identical_count,
missing_count, raw_results)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (
name, description,
json.dumps(cluster1_config),
json.dumps(cluster2_config),
json.dumps(query_options),
json.dumps(query_keys),
json.dumps(results_summary),
execution_time,
total_keys,
different_count,
identical_count,
missing_count,
json.dumps(raw_results) if raw_results else None
))
# 获取插入记录的ID
history_id = cursor.lastrowid
conn.commit()
logger.info(f"Redis查询历史记录 '{name}' 保存成功ID{history_id}")
return history_id
except Exception as e:
logger.error(f"保存Redis查询历史记录失败: {e}")
return None
finally:
conn.close()
def get_redis_query_history():
"""获取Redis查询历史记录"""
if not ensure_database():
logger.error("数据库初始化失败")
return []
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('''
SELECT id, name, description, execution_time, total_keys,
different_count, identical_count, missing_count, created_at
FROM redis_query_history
ORDER BY created_at DESC
''')
rows = cursor.fetchall()
history_list = []
for row in rows:
history_list.append({
'id': row['id'],
'name': row['name'],
'description': row['description'],
'execution_time': row['execution_time'],
'total_keys': row['total_keys'],
'different_count': row['different_count'],
'identical_count': row['identical_count'],
'missing_count': row['missing_count'],
'created_at': row['created_at']
})
return history_list
except Exception as e:
logger.error(f"获取Redis查询历史记录失败: {e}")
return []
finally:
conn.close()
def get_redis_query_history_by_id(history_id):
"""根据ID获取Redis查询历史记录详情"""
if not ensure_database():
logger.error("数据库初始化失败")
return None
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('''
SELECT * FROM redis_query_history WHERE id = ?
''', (history_id,))
row = cursor.fetchone()
if row:
return {
'id': row['id'],
'name': row['name'],
'description': row['description'],
'cluster1_config': json.loads(row['cluster1_config']),
'cluster2_config': json.loads(row['cluster2_config']),
'query_options': json.loads(row['query_options']),
'query_keys': json.loads(row['query_keys']),
'results_summary': json.loads(row['results_summary']),
'execution_time': row['execution_time'],
'total_keys': row['total_keys'],
'different_count': row['different_count'],
'identical_count': row['identical_count'],
'missing_count': row['missing_count'],
'created_at': row['created_at'],
'raw_results': json.loads(row['raw_results']) if row['raw_results'] else None
}
return None
except Exception as e:
logger.error(f"获取Redis查询历史记录详情失败: {e}")
return None
finally:
conn.close()
def delete_redis_query_history(history_id):
"""删除Redis查询历史记录"""
if not ensure_database():
logger.error("数据库初始化失败")
return False
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('DELETE FROM redis_query_history WHERE id = ?', (history_id,))
conn.commit()
success = cursor.rowcount > 0
if success:
logger.info(f"Redis查询历史记录ID {history_id} 删除成功")
return success
except Exception as e:
logger.error(f"删除Redis查询历史记录失败: {e}")
return False
finally:
conn.close()
def parse_redis_config_from_yaml(yaml_text):
"""从YAML格式文本解析Redis配置"""
try:
config = {}
lines = yaml_text.strip().split('\n')
for line in lines:
line = line.strip()
if ':' in line:
key, value = line.split(':', 1)
key = key.strip()
value = value.strip()
# 移除引号
if value.startswith('"') and value.endswith('"'):
value = value[1:-1]
elif value.startswith("'") and value.endswith("'"):
value = value[1:-1]
config[key] = value
# 转换为Redis集群配置格式
redis_config = {
'name': config.get('clusterName', ''),
'nodes': [],
'password': config.get('clusterPassword', ''),
'socket_timeout': 3,
'socket_connect_timeout': 3,
'max_connections_per_node': 16
}
# 解析地址
cluster_address = config.get('clusterAddress', '')
if cluster_address:
if ':' in cluster_address:
host, port = cluster_address.split(':', 1)
redis_config['nodes'] = [{'host': host, 'port': int(port)}]
else:
redis_config['nodes'] = [{'host': cluster_address, 'port': 6379}]
return redis_config
except Exception as e:
logger.error(f"解析Redis配置失败: {e}")
return None
def save_config_group(name, description, pro_config, test_config, query_config, sharding_config=None):
"""保存配置组"""
if not ensure_database():
logger.error("数据库初始化失败")
return False
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('''
INSERT OR REPLACE INTO config_groups
(name, description, pro_config, test_config, query_config, sharding_config, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', (
name, description,
json.dumps(pro_config),
json.dumps(test_config),
json.dumps(query_config),
json.dumps(sharding_config) if sharding_config else None,
datetime.now().isoformat()
))
conn.commit()
logger.info(f"配置组 '{name}' 保存成功,包含分表配置: {sharding_config is not None}")
return True
except Exception as e:
logger.error(f"保存配置组失败: {e}")
return False
finally:
conn.close()
def get_config_groups():
"""获取所有配置组"""
if not ensure_database():
logger.error("数据库初始化失败")
return []
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('''
SELECT id, name, description, created_at, updated_at
FROM config_groups
ORDER BY updated_at DESC
''')
rows = cursor.fetchall()
config_groups = []
for row in rows:
config_groups.append({
'id': row['id'],
'name': row['name'],
'description': row['description'],
'created_at': row['created_at'],
'updated_at': row['updated_at']
})
return config_groups
except Exception as e:
logger.error(f"获取配置组失败: {e}")
return []
finally:
conn.close()
def get_config_group_by_id(group_id):
"""根据ID获取配置组详情"""
if not ensure_database():
logger.error("数据库初始化失败")
return None
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('''
SELECT id, name, description, pro_config, test_config, query_config,
sharding_config, created_at, updated_at
FROM config_groups WHERE id = ?
''', (group_id,))
row = cursor.fetchone()
if row:
config = {
'id': row['id'],
'name': row['name'],
'description': row['description'],
'pro_config': json.loads(row['pro_config']),
'test_config': json.loads(row['test_config']),
'query_config': json.loads(row['query_config']),
'created_at': row['created_at'],
'updated_at': row['updated_at']
}
# 添加分表配置
if row['sharding_config']:
try:
config['sharding_config'] = json.loads(row['sharding_config'])
except (json.JSONDecodeError, TypeError):
config['sharding_config'] = None
else:
config['sharding_config'] = None
return config
return None
except Exception as e:
logger.error(f"获取配置组详情失败: {e}")
return None
finally:
conn.close()
def delete_config_group(group_id):
"""删除配置组"""
if not ensure_database():
logger.error("数据库初始化失败")
return False
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('DELETE FROM config_groups WHERE id = ?', (group_id,))
conn.commit()
success = cursor.rowcount > 0
if success:
logger.info(f"配置组ID {group_id} 删除成功")
return success
except Exception as e:
logger.error(f"删除配置组失败: {e}")
return False
finally:
conn.close()
def save_query_history(name, description, pro_config, test_config, query_config, query_keys,
results_summary, execution_time, total_keys, differences_count, identical_count,
sharding_config=None, query_type='single', raw_results=None, differences_data=None, identical_data=None):
"""保存查询历史记录支持分表查询和查询结果数据返回历史记录ID"""
if not ensure_database():
logger.error("数据库初始化失败")
return None
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('''
INSERT INTO query_history
(name, description, pro_config, test_config, query_config, query_keys,
results_summary, execution_time, total_keys, differences_count, identical_count,
sharding_config, query_type, raw_results, differences_data, identical_data)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (
name, description,
json.dumps(pro_config),
json.dumps(test_config),
json.dumps(query_config),
json.dumps(query_keys),
json.dumps(results_summary),
execution_time,
total_keys,
differences_count,
identical_count,
json.dumps(sharding_config) if sharding_config else None,
query_type,
json.dumps(raw_results) if raw_results else None,
json.dumps(differences_data) if differences_data else None,
json.dumps(identical_data) if identical_data else None
))
# 获取插入记录的ID
history_id = cursor.lastrowid
conn.commit()
logger.info(f"查询历史记录 '{name}' 保存成功,查询类型:{query_type}ID{history_id}")
return history_id
except Exception as e:
logger.error(f"保存查询历史记录失败: {e}")
return None
finally:
conn.close()
def get_query_history():
"""获取所有查询历史记录"""
if not ensure_database():
logger.error("数据库初始化失败")
return []
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('''
SELECT id, name, description, execution_time, total_keys,
differences_count, identical_count, created_at, query_type
FROM query_history
ORDER BY created_at DESC
''')
rows = cursor.fetchall()
history_list = []
for row in rows:
# 获取列名列表以检查字段是否存在
column_names = [desc[0] for desc in cursor.description]
history_list.append({
'id': row['id'],
'name': row['name'],
'description': row['description'],
'execution_time': row['execution_time'],
'total_keys': row['total_keys'],
'differences_count': row['differences_count'],
'identical_count': row['identical_count'],
'created_at': row['created_at'],
'query_type': row['query_type'] if 'query_type' in column_names else 'single'
})
return history_list
except Exception as e:
logger.error(f"获取查询历史记录失败: {e}")
return []
finally:
conn.close()
def get_query_history_by_id(history_id):
"""根据ID获取查询历史记录详情"""
if not ensure_database():
logger.error("数据库初始化失败")
return None
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('''
SELECT * FROM query_history WHERE id = ?
''', (history_id,))
row = cursor.fetchone()
if row:
# 获取列名列表以检查字段是否存在
column_names = [desc[0] for desc in cursor.description]
return {
'id': row['id'],
'name': row['name'],
'description': row['description'],
'pro_config': json.loads(row['pro_config']),
'test_config': json.loads(row['test_config']),
'query_config': json.loads(row['query_config']),
'query_keys': json.loads(row['query_keys']),
'results_summary': json.loads(row['results_summary']),
'execution_time': row['execution_time'],
'total_keys': row['total_keys'],
'differences_count': row['differences_count'],
'identical_count': row['identical_count'],
'created_at': row['created_at'],
# 处理新字段,保持向后兼容
'sharding_config': json.loads(row['sharding_config']) if 'sharding_config' in column_names and row['sharding_config'] else None,
'query_type': row['query_type'] if 'query_type' in column_names else 'single',
# 添加查询结果数据支持
'raw_results': json.loads(row['raw_results']) if 'raw_results' in column_names and row['raw_results'] else None,
'differences_data': json.loads(row['differences_data']) if 'differences_data' in column_names and row['differences_data'] else None,
'identical_data': json.loads(row['identical_data']) if 'identical_data' in column_names and row['identical_data'] else None
}
return None
except Exception as e:
logger.error(f"获取查询历史记录详情失败: {e}")
return None
finally:
conn.close()
def delete_query_history(history_id):
"""删除查询历史记录"""
if not ensure_database():
logger.error("数据库初始化失败")
return False
conn = get_db_connection()
cursor = conn.cursor()
try:
cursor.execute('DELETE FROM query_history WHERE id = ?', (history_id,))
conn.commit()
success = cursor.rowcount > 0
if success:
logger.info(f"查询历史记录ID {history_id} 删除成功")
return success
except Exception as e:
logger.error(f"删除查询历史记录失败: {e}")
return False
finally:
conn.close()