228 lines
9.4 KiB
Python
228 lines
9.4 KiB
Python
"""
|
|
数据库管理模块
|
|
负责SQLite数据库的初始化、连接和表结构管理
|
|
"""
|
|
|
|
import sqlite3
|
|
import json
|
|
import os
|
|
import logging
|
|
from datetime import datetime
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DATABASE_PATH = 'config_groups.db'
|
|
|
|
def init_database():
|
|
"""初始化数据库"""
|
|
try:
|
|
conn = sqlite3.connect(DATABASE_PATH)
|
|
cursor = conn.cursor()
|
|
|
|
# 创建配置组表
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS config_groups (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT NOT NULL UNIQUE,
|
|
description TEXT,
|
|
pro_config TEXT NOT NULL,
|
|
test_config TEXT NOT NULL,
|
|
query_config TEXT NOT NULL,
|
|
sharding_config TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
''')
|
|
|
|
# 创建查询历史表，包含分表配置字段
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS query_history (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT NOT NULL,
|
|
description TEXT,
|
|
pro_config TEXT NOT NULL,
|
|
test_config TEXT NOT NULL,
|
|
query_config TEXT NOT NULL,
|
|
query_keys TEXT NOT NULL,
|
|
results_summary TEXT NOT NULL,
|
|
execution_time REAL NOT NULL,
|
|
total_keys INTEGER NOT NULL,
|
|
differences_count INTEGER NOT NULL,
|
|
identical_count INTEGER NOT NULL,
|
|
sharding_config TEXT,
|
|
query_type TEXT DEFAULT 'single',
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
''')
|
|
|
|
# 创建分表配置组表
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS sharding_config_groups (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT NOT NULL UNIQUE,
|
|
description TEXT,
|
|
pro_config TEXT NOT NULL,
|
|
test_config TEXT NOT NULL,
|
|
query_config TEXT NOT NULL,
|
|
sharding_config TEXT NOT NULL,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
''')
|
|
|
|
# 创建查询日志表
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS query_logs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
batch_id TEXT NOT NULL,
|
|
history_id INTEGER,
|
|
timestamp TEXT NOT NULL,
|
|
level TEXT NOT NULL,
|
|
message TEXT NOT NULL,
|
|
query_type TEXT DEFAULT 'single',
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (history_id) REFERENCES query_history (id) ON DELETE CASCADE
|
|
)
|
|
''')
|
|
|
|
# 创建Redis配置组表
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS redis_config_groups (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT NOT NULL UNIQUE,
|
|
description TEXT,
|
|
cluster1_config TEXT NOT NULL,
|
|
cluster2_config TEXT NOT NULL,
|
|
query_options TEXT NOT NULL,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
''')
|
|
|
|
# 创建Redis查询历史表
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS redis_query_history (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT NOT NULL,
|
|
description TEXT,
|
|
cluster1_config TEXT NOT NULL,
|
|
cluster2_config TEXT NOT NULL,
|
|
query_options TEXT NOT NULL,
|
|
query_keys TEXT NOT NULL,
|
|
results_summary TEXT NOT NULL,
|
|
execution_time REAL NOT NULL,
|
|
total_keys INTEGER NOT NULL,
|
|
different_count INTEGER NOT NULL,
|
|
identical_count INTEGER NOT NULL,
|
|
missing_count INTEGER NOT NULL,
|
|
raw_results TEXT,
|
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
''')
|
|
|
|
# 创建索引
|
|
cursor.execute('CREATE INDEX IF NOT EXISTS idx_query_logs_batch_id ON query_logs(batch_id)')
|
|
cursor.execute('CREATE INDEX IF NOT EXISTS idx_query_logs_history_id ON query_logs(history_id)')
|
|
cursor.execute('CREATE INDEX IF NOT EXISTS idx_query_logs_timestamp ON query_logs(timestamp)')
|
|
cursor.execute('CREATE INDEX IF NOT EXISTS idx_query_logs_level ON query_logs(level)')
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
logger.info("数据库初始化完成")
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"数据库初始化失败: {e}")
|
|
return False
|
|
|
|
def ensure_database():
|
|
"""确保数据库和表存在"""
|
|
if not os.path.exists(DATABASE_PATH):
|
|
logger.info("数据库文件不存在，正在创建...")
|
|
return init_database()
|
|
|
|
# 检查表是否存在
|
|
try:
|
|
conn = sqlite3.connect(DATABASE_PATH)
|
|
cursor = conn.cursor()
|
|
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name IN ('config_groups', 'query_history', 'sharding_config_groups', 'query_logs', 'redis_config_groups', 'redis_query_history')")
|
|
results = cursor.fetchall()
|
|
existing_tables = [row[0] for row in results]
|
|
|
|
required_tables = ['config_groups', 'query_history', 'sharding_config_groups', 'query_logs', 'redis_config_groups', 'redis_query_history']
|
|
missing_tables = [table for table in required_tables if table not in existing_tables]
|
|
|
|
if missing_tables:
|
|
logger.info(f"数据库表不完整，缺少表：{missing_tables}，正在重新创建...")
|
|
return init_database()
|
|
|
|
# 检查config_groups表是否有sharding_config字段
|
|
cursor.execute("PRAGMA table_info(config_groups)")
|
|
columns = cursor.fetchall()
|
|
column_names = [column[1] for column in columns]
|
|
|
|
if 'sharding_config' not in column_names:
|
|
logger.info("添加sharding_config字段到config_groups表...")
|
|
cursor.execute("ALTER TABLE config_groups ADD COLUMN sharding_config TEXT")
|
|
conn.commit()
|
|
logger.info("sharding_config字段添加成功")
|
|
|
|
# 检查query_history表是否有分表相关字段
|
|
cursor.execute("PRAGMA table_info(query_history)")
|
|
history_columns = cursor.fetchall()
|
|
history_column_names = [column[1] for column in history_columns]
|
|
|
|
if 'sharding_config' not in history_column_names:
|
|
logger.info("添加sharding_config字段到query_history表...")
|
|
cursor.execute("ALTER TABLE query_history ADD COLUMN sharding_config TEXT")
|
|
conn.commit()
|
|
logger.info("query_history表sharding_config字段添加成功")
|
|
|
|
if 'query_type' not in history_column_names:
|
|
logger.info("添加query_type字段到query_history表...")
|
|
cursor.execute("ALTER TABLE query_history ADD COLUMN query_type TEXT DEFAULT 'single'")
|
|
conn.commit()
|
|
logger.info("query_history表query_type字段添加成功")
|
|
|
|
# 添加查询结果数据存储字段
|
|
if 'raw_results' not in history_column_names:
|
|
logger.info("添加raw_results字段到query_history表...")
|
|
cursor.execute("ALTER TABLE query_history ADD COLUMN raw_results TEXT")
|
|
conn.commit()
|
|
logger.info("query_history表raw_results字段添加成功")
|
|
|
|
if 'differences_data' not in history_column_names:
|
|
logger.info("添加differences_data字段到query_history表...")
|
|
cursor.execute("ALTER TABLE query_history ADD COLUMN differences_data TEXT")
|
|
conn.commit()
|
|
logger.info("query_history表differences_data字段添加成功")
|
|
|
|
if 'identical_data' not in history_column_names:
|
|
logger.info("添加identical_data字段到query_history表...")
|
|
cursor.execute("ALTER TABLE query_history ADD COLUMN identical_data TEXT")
|
|
conn.commit()
|
|
logger.info("query_history表identical_data字段添加成功")
|
|
|
|
# 检查query_logs表是否存在history_id字段
|
|
cursor.execute("PRAGMA table_info(query_logs)")
|
|
logs_columns = cursor.fetchall()
|
|
logs_column_names = [column[1] for column in logs_columns]
|
|
|
|
if 'history_id' not in logs_column_names:
|
|
logger.info("添加history_id字段到query_logs表...")
|
|
cursor.execute("ALTER TABLE query_logs ADD COLUMN history_id INTEGER")
|
|
# 创建外键索引
|
|
cursor.execute('CREATE INDEX IF NOT EXISTS idx_query_logs_history_id ON query_logs(history_id)')
|
|
conn.commit()
|
|
logger.info("query_logs表history_id字段添加成功")
|
|
|
|
conn.close()
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"检查数据库表失败: {e}")
|
|
return init_database()
|
|
|
|
def get_db_connection():
|
|
"""获取数据库连接"""
|
|
conn = sqlite3.connect(DATABASE_PATH)
|
|
conn.row_factory = sqlite3.Row
|
|
return conn |