增加Redis查询比对
This commit is contained in:
228
modules/database.py
Normal file
228
modules/database.py
Normal file
@@ -0,0 +1,228 @@
|
||||
"""
|
||||
数据库管理模块
|
||||
负责SQLite数据库的初始化、连接和表结构管理
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
import json
|
||||
import os
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DATABASE_PATH = 'config_groups.db'
|
||||
|
||||
def init_database():
|
||||
"""初始化数据库"""
|
||||
try:
|
||||
conn = sqlite3.connect(DATABASE_PATH)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# 创建配置组表
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS config_groups (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
description TEXT,
|
||||
pro_config TEXT NOT NULL,
|
||||
test_config TEXT NOT NULL,
|
||||
query_config TEXT NOT NULL,
|
||||
sharding_config TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
''')
|
||||
|
||||
# 创建查询历史表，包含分表配置字段
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS query_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
pro_config TEXT NOT NULL,
|
||||
test_config TEXT NOT NULL,
|
||||
query_config TEXT NOT NULL,
|
||||
query_keys TEXT NOT NULL,
|
||||
results_summary TEXT NOT NULL,
|
||||
execution_time REAL NOT NULL,
|
||||
total_keys INTEGER NOT NULL,
|
||||
differences_count INTEGER NOT NULL,
|
||||
identical_count INTEGER NOT NULL,
|
||||
sharding_config TEXT,
|
||||
query_type TEXT DEFAULT 'single',
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
''')
|
||||
|
||||
# 创建分表配置组表
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS sharding_config_groups (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
description TEXT,
|
||||
pro_config TEXT NOT NULL,
|
||||
test_config TEXT NOT NULL,
|
||||
query_config TEXT NOT NULL,
|
||||
sharding_config TEXT NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
''')
|
||||
|
||||
# 创建查询日志表
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS query_logs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
batch_id TEXT NOT NULL,
|
||||
history_id INTEGER,
|
||||
timestamp TEXT NOT NULL,
|
||||
level TEXT NOT NULL,
|
||||
message TEXT NOT NULL,
|
||||
query_type TEXT DEFAULT 'single',
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (history_id) REFERENCES query_history (id) ON DELETE CASCADE
|
||||
)
|
||||
''')
|
||||
|
||||
# 创建Redis配置组表
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS redis_config_groups (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
description TEXT,
|
||||
cluster1_config TEXT NOT NULL,
|
||||
cluster2_config TEXT NOT NULL,
|
||||
query_options TEXT NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
''')
|
||||
|
||||
# 创建Redis查询历史表
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS redis_query_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
cluster1_config TEXT NOT NULL,
|
||||
cluster2_config TEXT NOT NULL,
|
||||
query_options TEXT NOT NULL,
|
||||
query_keys TEXT NOT NULL,
|
||||
results_summary TEXT NOT NULL,
|
||||
execution_time REAL NOT NULL,
|
||||
total_keys INTEGER NOT NULL,
|
||||
different_count INTEGER NOT NULL,
|
||||
identical_count INTEGER NOT NULL,
|
||||
missing_count INTEGER NOT NULL,
|
||||
raw_results TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
''')
|
||||
|
||||
# 创建索引
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_query_logs_batch_id ON query_logs(batch_id)')
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_query_logs_history_id ON query_logs(history_id)')
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_query_logs_timestamp ON query_logs(timestamp)')
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_query_logs_level ON query_logs(level)')
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
logger.info("数据库初始化完成")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"数据库初始化失败: {e}")
|
||||
return False
|
||||
|
||||
def ensure_database():
|
||||
"""确保数据库和表存在"""
|
||||
if not os.path.exists(DATABASE_PATH):
|
||||
logger.info("数据库文件不存在，正在创建...")
|
||||
return init_database()
|
||||
|
||||
# 检查表是否存在
|
||||
try:
|
||||
conn = sqlite3.connect(DATABASE_PATH)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name IN ('config_groups', 'query_history', 'sharding_config_groups', 'query_logs', 'redis_config_groups', 'redis_query_history')")
|
||||
results = cursor.fetchall()
|
||||
existing_tables = [row[0] for row in results]
|
||||
|
||||
required_tables = ['config_groups', 'query_history', 'sharding_config_groups', 'query_logs', 'redis_config_groups', 'redis_query_history']
|
||||
missing_tables = [table for table in required_tables if table not in existing_tables]
|
||||
|
||||
if missing_tables:
|
||||
logger.info(f"数据库表不完整，缺少表：{missing_tables}，正在重新创建...")
|
||||
return init_database()
|
||||
|
||||
# 检查config_groups表是否有sharding_config字段
|
||||
cursor.execute("PRAGMA table_info(config_groups)")
|
||||
columns = cursor.fetchall()
|
||||
column_names = [column[1] for column in columns]
|
||||
|
||||
if 'sharding_config' not in column_names:
|
||||
logger.info("添加sharding_config字段到config_groups表...")
|
||||
cursor.execute("ALTER TABLE config_groups ADD COLUMN sharding_config TEXT")
|
||||
conn.commit()
|
||||
logger.info("sharding_config字段添加成功")
|
||||
|
||||
# 检查query_history表是否有分表相关字段
|
||||
cursor.execute("PRAGMA table_info(query_history)")
|
||||
history_columns = cursor.fetchall()
|
||||
history_column_names = [column[1] for column in history_columns]
|
||||
|
||||
if 'sharding_config' not in history_column_names:
|
||||
logger.info("添加sharding_config字段到query_history表...")
|
||||
cursor.execute("ALTER TABLE query_history ADD COLUMN sharding_config TEXT")
|
||||
conn.commit()
|
||||
logger.info("query_history表sharding_config字段添加成功")
|
||||
|
||||
if 'query_type' not in history_column_names:
|
||||
logger.info("添加query_type字段到query_history表...")
|
||||
cursor.execute("ALTER TABLE query_history ADD COLUMN query_type TEXT DEFAULT 'single'")
|
||||
conn.commit()
|
||||
logger.info("query_history表query_type字段添加成功")
|
||||
|
||||
# 添加查询结果数据存储字段
|
||||
if 'raw_results' not in history_column_names:
|
||||
logger.info("添加raw_results字段到query_history表...")
|
||||
cursor.execute("ALTER TABLE query_history ADD COLUMN raw_results TEXT")
|
||||
conn.commit()
|
||||
logger.info("query_history表raw_results字段添加成功")
|
||||
|
||||
if 'differences_data' not in history_column_names:
|
||||
logger.info("添加differences_data字段到query_history表...")
|
||||
cursor.execute("ALTER TABLE query_history ADD COLUMN differences_data TEXT")
|
||||
conn.commit()
|
||||
logger.info("query_history表differences_data字段添加成功")
|
||||
|
||||
if 'identical_data' not in history_column_names:
|
||||
logger.info("添加identical_data字段到query_history表...")
|
||||
cursor.execute("ALTER TABLE query_history ADD COLUMN identical_data TEXT")
|
||||
conn.commit()
|
||||
logger.info("query_history表identical_data字段添加成功")
|
||||
|
||||
# 检查query_logs表是否存在history_id字段
|
||||
cursor.execute("PRAGMA table_info(query_logs)")
|
||||
logs_columns = cursor.fetchall()
|
||||
logs_column_names = [column[1] for column in logs_columns]
|
||||
|
||||
if 'history_id' not in logs_column_names:
|
||||
logger.info("添加history_id字段到query_logs表...")
|
||||
cursor.execute("ALTER TABLE query_logs ADD COLUMN history_id INTEGER")
|
||||
# 创建外键索引
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_query_logs_history_id ON query_logs(history_id)')
|
||||
conn.commit()
|
||||
logger.info("query_logs表history_id字段添加成功")
|
||||
|
||||
conn.close()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"检查数据库表失败: {e}")
|
||||
return init_database()
|
||||
|
||||
def get_db_connection():
|
||||
"""获取数据库连接"""
|
||||
conn = sqlite3.connect(DATABASE_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
Reference in New Issue
Block a user