增加分表查询
This commit is contained in:
64
app.py
64
app.py
@@ -1175,7 +1175,7 @@ def create_connection(config):
|
|||||||
return None, None
|
return None, None
|
||||||
|
|
||||||
def execute_query(session, table, keys, fields, values, exclude_fields=None):
|
def execute_query(session, table, keys, fields, values, exclude_fields=None):
|
||||||
"""执行查询"""
|
"""执行查询,支持单主键和复合主键"""
|
||||||
try:
|
try:
|
||||||
# 参数验证
|
# 参数验证
|
||||||
if not keys or len(keys) == 0:
|
if not keys or len(keys) == 0:
|
||||||
@@ -1187,8 +1187,37 @@ def execute_query(session, table, keys, fields, values, exclude_fields=None):
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
# 构建查询条件
|
# 构建查询条件
|
||||||
quoted_values = [f"'{value}'" for value in values]
|
if len(keys) == 1:
|
||||||
query_conditions = f"{keys[0]} IN ({', '.join(quoted_values)})"
|
# 单主键查询(保持原有逻辑)
|
||||||
|
quoted_values = [f"'{value}'" for value in values]
|
||||||
|
query_conditions = f"{keys[0]} IN ({', '.join(quoted_values)})"
|
||||||
|
else:
|
||||||
|
# 复合主键查询
|
||||||
|
conditions = []
|
||||||
|
for value in values:
|
||||||
|
# 检查value是否包含复合主键分隔符
|
||||||
|
if isinstance(value, str) and ',' in value:
|
||||||
|
# 解析复合主键值
|
||||||
|
key_values = [v.strip() for v in value.split(',')]
|
||||||
|
if len(key_values) == len(keys):
|
||||||
|
# 构建单个复合主键条件: (key1='val1' AND key2='val2')
|
||||||
|
key_conditions = []
|
||||||
|
for i, (key, val) in enumerate(zip(keys, key_values)):
|
||||||
|
key_conditions.append(f"{key} = '{val}'")
|
||||||
|
conditions.append(f"({' AND '.join(key_conditions)})")
|
||||||
|
else:
|
||||||
|
logger.warning(f"复合主键值 '{value}' 的字段数量({len(key_values)})与主键字段数量({len(keys)})不匹配")
|
||||||
|
# 将其作为第一个主键的值处理
|
||||||
|
conditions.append(f"{keys[0]} = '{value}'")
|
||||||
|
else:
|
||||||
|
# 单值,作为第一个主键的值处理
|
||||||
|
conditions.append(f"{keys[0]} = '{value}'")
|
||||||
|
|
||||||
|
if conditions:
|
||||||
|
query_conditions = ' OR '.join(conditions)
|
||||||
|
else:
|
||||||
|
logger.error("无法构建有效的查询条件")
|
||||||
|
return []
|
||||||
|
|
||||||
# 确定要查询的字段
|
# 确定要查询的字段
|
||||||
if fields:
|
if fields:
|
||||||
@@ -1200,7 +1229,10 @@ def execute_query(session, table, keys, fields, values, exclude_fields=None):
|
|||||||
|
|
||||||
# 记录查询SQL日志
|
# 记录查询SQL日志
|
||||||
logger.info(f"执行查询SQL: {query_sql}")
|
logger.info(f"执行查询SQL: {query_sql}")
|
||||||
logger.info(f"查询参数: 表={table}, 字段={fields_str}, Key数量={len(values)}")
|
if len(keys) > 1:
|
||||||
|
logger.info(f"复合主键查询参数: 表={table}, 主键字段={keys}, 字段={fields_str}, Key数量={len(values)}")
|
||||||
|
else:
|
||||||
|
logger.info(f"单主键查询参数: 表={table}, 主键字段={keys[0]}, 字段={fields_str}, Key数量={len(values)}")
|
||||||
|
|
||||||
# 执行查询
|
# 执行查询
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
@@ -1367,15 +1399,29 @@ def execute_mixed_query(pro_session, test_session, pro_config, test_config, keys
|
|||||||
return results
|
return results
|
||||||
|
|
||||||
def compare_results(pro_data, test_data, keys, fields_to_compare, exclude_fields, values):
|
def compare_results(pro_data, test_data, keys, fields_to_compare, exclude_fields, values):
|
||||||
"""比较查询结果"""
|
"""比较查询结果,支持复合主键"""
|
||||||
differences = []
|
differences = []
|
||||||
field_diff_count = {}
|
field_diff_count = {}
|
||||||
identical_results = [] # 存储相同的结果
|
identical_results = [] # 存储相同的结果
|
||||||
|
|
||||||
|
def match_composite_key(row, composite_value, keys):
|
||||||
|
"""检查数据行是否匹配复合主键值"""
|
||||||
|
if len(keys) == 1:
|
||||||
|
# 单主键匹配
|
||||||
|
return getattr(row, keys[0]) == composite_value
|
||||||
|
else:
|
||||||
|
# 复合主键匹配
|
||||||
|
if isinstance(composite_value, str) and ',' in composite_value:
|
||||||
|
key_values = [v.strip() for v in composite_value.split(',')]
|
||||||
|
if len(key_values) == len(keys):
|
||||||
|
return all(str(getattr(row, key)) == key_val for key, key_val in zip(keys, key_values))
|
||||||
|
# 如果不是复合值,只匹配第一个主键
|
||||||
|
return getattr(row, keys[0]) == composite_value
|
||||||
|
|
||||||
for value in values:
|
for value in values:
|
||||||
# 查找原表和测试表中该ID的相关数据
|
# 查找生产表和测试表中该主键值的相关数据
|
||||||
rows_pro = [row for row in pro_data if getattr(row, keys[0]) == value]
|
rows_pro = [row for row in pro_data if match_composite_key(row, value, keys)]
|
||||||
rows_test = [row for row in test_data if getattr(row, keys[0]) == value]
|
rows_test = [row for row in test_data if match_composite_key(row, value, keys)]
|
||||||
|
|
||||||
for row_pro in rows_pro:
|
for row_pro in rows_pro:
|
||||||
# 在测试表中查找相同主键的行
|
# 在测试表中查找相同主键的行
|
||||||
@@ -2180,4 +2226,4 @@ def api_get_query_logs_by_history(history_id):
|
|||||||
return jsonify({'success': False, 'error': str(e)}), 500
|
return jsonify({'success': False, 'error': str(e)}), 500
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
app.run(debug=True, port=5000)
|
app.run(debug=True, port=5001)
|
||||||
|
@@ -52,24 +52,24 @@ function toggleShardingMode() {
|
|||||||
// 启用分表模式
|
// 启用分表模式
|
||||||
shardingConfig.style.display = 'block';
|
shardingConfig.style.display = 'block';
|
||||||
executeButton.textContent = '执行分表查询比对';
|
executeButton.textContent = '执行分表查询比对';
|
||||||
keyInputHint.textContent = '分表模式:Key值应包含时间戳用于计算分表索引';
|
keyInputHint.textContent = '分表模式:Key值应包含时间戳用于计算分表索引,支持单主键和复合主键(逗号分隔)';
|
||||||
keysField.placeholder = ' (推荐使用包含时间戳的字段)';
|
keysField.placeholder = '单主键:wmid 或 复合主键:docid,id (推荐使用包含时间戳的字段)';
|
||||||
keysField.value = '';
|
keysField.value = '';
|
||||||
|
|
||||||
// 更新查询Key输入框的占位符
|
// 更新查询Key输入框的占位符
|
||||||
const queryValues = document.getElementById('query_values');
|
const queryValues = document.getElementById('query_values');
|
||||||
queryValues.placeholder = '请输入查询的Key值,一行一个\n分表查询示例(包含时间戳):\nwmid_1609459200\nwmid_1610064000\nwmid_1610668800';
|
queryValues.placeholder = '请输入查询的Key值,一行一个\n单主键示例(包含时间戳):\nwmid_1609459200\nwmid_1610064000\nwmid_1610668800\n\n复合主键示例(逗号分隔):\ndocid_1609459200,id1\ndocid_1610064000,id2\ndocid_1610668800,id3';
|
||||||
} else {
|
} else {
|
||||||
// 禁用分表模式
|
// 禁用分表模式
|
||||||
shardingConfig.style.display = 'none';
|
shardingConfig.style.display = 'none';
|
||||||
executeButton.textContent = '执行查询比对';
|
executeButton.textContent = '执行查询比对';
|
||||||
keyInputHint.textContent = '单表模式:输入普通Key值';
|
keyInputHint.textContent = '单表模式:支持单主键和复合主键查询,复合主键用逗号分隔';
|
||||||
keysField.placeholder = '';
|
keysField.placeholder = '单主键:docid 或 复合主键:docid,id';
|
||||||
keysField.value = '';
|
keysField.value = '';
|
||||||
|
|
||||||
// 更新查询Key输入框的占位符
|
// 更新查询Key输入框的占位符
|
||||||
const queryValues = document.getElementById('query_values');
|
const queryValues = document.getElementById('query_values');
|
||||||
queryValues.placeholder = '请输入查询的Key值,一行一个\n单表查询示例:\nkey1\nkey2\nkey3';
|
queryValues.placeholder = '请输入查询的Key值,一行一个\n单主键示例:\nkey1\nkey2\nkey3\n\n复合主键示例(逗号分隔):\ndocid1,id1\ndocid2,id2\ndocid3,id3';
|
||||||
}
|
}
|
||||||
|
|
||||||
updateTableNameHints();
|
updateTableNameHints();
|
||||||
@@ -975,7 +975,7 @@ function displayDifferences() {
|
|||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<p class="mb-0 mt-2"><strong>主键:</strong> ${JSON.stringify(diff.key)}</p>
|
<p class="mb-0 mt-2"><strong>主键:</strong> ${formatCompositeKey(diff.key)}</p>
|
||||||
</div>
|
</div>
|
||||||
<div class="card-body">
|
<div class="card-body">
|
||||||
<p class="text-warning"><i class="fas fa-exclamation-triangle"></i> ${diff.message}</p>
|
<p class="text-warning"><i class="fas fa-exclamation-triangle"></i> ${diff.message}</p>
|
||||||
@@ -1008,7 +1008,7 @@ function displayDifferences() {
|
|||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<p class="mb-0 mt-2"><strong>主键:</strong> ${JSON.stringify(diff.key)}</p>
|
<p class="mb-0 mt-2"><strong>主键:</strong> ${formatCompositeKey(diff.key)}</p>
|
||||||
<p class="mb-0"><strong>差异字段:</strong> ${diff.field}</p>
|
<p class="mb-0"><strong>差异字段:</strong> ${diff.field}</p>
|
||||||
</div>
|
</div>
|
||||||
<div class="collapse" id="diffCollapse${globalIndex}">
|
<div class="collapse" id="diffCollapse${globalIndex}">
|
||||||
@@ -1137,7 +1137,7 @@ function displayIdenticalResults() {
|
|||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<p class="mb-0 mt-2"><strong>主键:</strong> ${JSON.stringify(result.key)}</p>
|
<p class="mb-0 mt-2"><strong>主键:</strong> ${formatCompositeKey(result.key)}</p>
|
||||||
</div>
|
</div>
|
||||||
<div class="collapse" id="collapse${globalIndex}">
|
<div class="collapse" id="collapse${globalIndex}">
|
||||||
<div class="card-body">
|
<div class="card-body">
|
||||||
@@ -1559,16 +1559,34 @@ function showRawData(keyStr) {
|
|||||||
// 查找生产环境数据
|
// 查找生产环境数据
|
||||||
if (currentResults.raw_pro_data) {
|
if (currentResults.raw_pro_data) {
|
||||||
proData = currentResults.raw_pro_data.find(item => {
|
proData = currentResults.raw_pro_data.find(item => {
|
||||||
// 比较主键
|
// 支持复合主键比较
|
||||||
return JSON.stringify(item[Object.keys(key)[0]]) === JSON.stringify(key[Object.keys(key)[0]]);
|
if (typeof key === 'object' && !Array.isArray(key)) {
|
||||||
|
// 复合主键情况:比较所有主键字段
|
||||||
|
return Object.keys(key).every(keyField =>
|
||||||
|
JSON.stringify(item[keyField]) === JSON.stringify(key[keyField])
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
// 单主键情况:保持原有逻辑
|
||||||
|
const keyField = Object.keys(key)[0];
|
||||||
|
return JSON.stringify(item[keyField]) === JSON.stringify(key[keyField]);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// 查找测试环境数据
|
// 查找测试环境数据
|
||||||
if (currentResults.raw_test_data) {
|
if (currentResults.raw_test_data) {
|
||||||
testData = currentResults.raw_test_data.find(item => {
|
testData = currentResults.raw_test_data.find(item => {
|
||||||
// 比较主键
|
// 支持复合主键比较
|
||||||
return JSON.stringify(item[Object.keys(key)[0]]) === JSON.stringify(key[Object.keys(key)[0]]);
|
if (typeof key === 'object' && !Array.isArray(key)) {
|
||||||
|
// 复合主键情况:比较所有主键字段
|
||||||
|
return Object.keys(key).every(keyField =>
|
||||||
|
JSON.stringify(item[keyField]) === JSON.stringify(key[keyField])
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
// 单主键情况:保持原有逻辑
|
||||||
|
const keyField = Object.keys(key)[0];
|
||||||
|
return JSON.stringify(item[keyField]) === JSON.stringify(key[keyField]);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3867,7 +3885,7 @@ function renderRawDataContent() {
|
|||||||
const globalIndex = startIndex + index + 1;
|
const globalIndex = startIndex + index + 1;
|
||||||
const envBadgeClass = item.environment === 'production' ? 'bg-primary' : 'bg-success';
|
const envBadgeClass = item.environment === 'production' ? 'bg-primary' : 'bg-success';
|
||||||
|
|
||||||
// 获取主键值用于标识 - 修复主键解析问题
|
// 获取主键值用于标识 - 修复主键解析问题,支持复合主键
|
||||||
let keyValue = 'N/A';
|
let keyValue = 'N/A';
|
||||||
|
|
||||||
// 从当前配置获取keys,因为query_config可能不在results中
|
// 从当前配置获取keys,因为query_config可能不在results中
|
||||||
@@ -3884,11 +3902,12 @@ function renderRawDataContent() {
|
|||||||
}).filter(val => val !== 'N/A');
|
}).filter(val => val !== 'N/A');
|
||||||
|
|
||||||
if (keyValues.length > 0) {
|
if (keyValues.length > 0) {
|
||||||
|
// 复合主键显示格式:docid1,id1 或 单主键显示:value1
|
||||||
keyValue = keyValues.join(', ');
|
keyValue = keyValues.join(', ');
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// 如果没有配置主键,尝试常见的主键字段名
|
// 如果没有配置主键,尝试常见的主键字段名
|
||||||
const commonKeyFields = ['id', 'statusid', 'key', 'uuid'];
|
const commonKeyFields = ['id', 'statusid', 'key', 'uuid', 'docid'];
|
||||||
for (const field of commonKeyFields) {
|
for (const field of commonKeyFields) {
|
||||||
if (item.data[field] !== undefined) {
|
if (item.data[field] !== undefined) {
|
||||||
keyValue = item.data[field];
|
keyValue = item.data[field];
|
||||||
@@ -4122,4 +4141,21 @@ function copyRawRecord(recordStr) {
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
showAlert('danger', '复制失败: ' + error.message);
|
showAlert('danger', '复制失败: ' + error.message);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 格式化复合主键显示
|
||||||
|
function formatCompositeKey(key) {
|
||||||
|
if (!key) {
|
||||||
|
return 'N/A';
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof key === 'object' && !Array.isArray(key)) {
|
||||||
|
// 复合主键情况:显示为 "docid: value1, id: value2" 格式
|
||||||
|
return Object.entries(key)
|
||||||
|
.map(([field, value]) => `${field}: ${value}`)
|
||||||
|
.join(', ');
|
||||||
|
} else {
|
||||||
|
// 单主键或其他情况:使用JSON格式
|
||||||
|
return JSON.stringify(key);
|
||||||
|
}
|
||||||
}
|
}
|
@@ -539,9 +539,9 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="card-body">
|
<div class="card-body">
|
||||||
<div class="mb-3">
|
<div class="mb-3">
|
||||||
<label class="form-label">主键字段 (逗号分隔)</label>
|
<label class="form-label">主键字段 (逗号分隔,支持复合主键)</label>
|
||||||
<input type="text" class="form-control form-control-sm" id="keys" placeholder="" value="">
|
<input type="text" class="form-control form-control-sm" id="keys" placeholder="单主键:docid 或 复合主键:docid,id" value="">
|
||||||
<small class="form-text text-muted">分表模式下推荐使用包含时间戳的字段如</small>
|
<small class="form-text text-muted">支持单主键或复合主键,复合主键用逗号分隔,如:docid,id</small>
|
||||||
</div>
|
</div>
|
||||||
<div class="mb-3">
|
<div class="mb-3">
|
||||||
<label class="form-label">比较字段 (空则比较全部,逗号分隔)</label>
|
<label class="form-label">比较字段 (空则比较全部,逗号分隔)</label>
|
||||||
@@ -562,7 +562,7 @@
|
|||||||
<h4><i class="fas fa-key"></i> 查询Key管理</h4>
|
<h4><i class="fas fa-key"></i> 查询Key管理</h4>
|
||||||
<div class="mb-3">
|
<div class="mb-3">
|
||||||
<label class="form-label">批量Key输入 (一行一个)</label>
|
<label class="form-label">批量Key输入 (一行一个)</label>
|
||||||
<textarea class="form-control query-keys" id="query_values" placeholder="请输入查询的Key值,一行一个 单表查询示例: key1 key2 key3 分表查询示例(包含时间戳): wmid_1609459200 wmid_1610064000 wmid_1610668800"></textarea>
|
<textarea class="form-control query-keys" id="query_values" placeholder="请输入查询的Key值,一行一个 单主键示例: key1 key2 key3 复合主键示例(逗号分隔): docid1,id1 docid2,id2 docid3,id3 分表查询示例(包含时间戳): wmid_1609459200 wmid_1610064000 wmid_1610668800"></textarea>
|
||||||
<small class="form-text text-muted" id="key_input_hint">单表模式:输入普通Key值 | 分表模式:Key值应包含时间戳用于计算分表索引</small>
|
<small class="form-text text-muted" id="key_input_hint">单表模式:输入普通Key值 | 分表模式:Key值应包含时间戳用于计算分表索引</small>
|
||||||
</div>
|
</div>
|
||||||
<div class="mb-3">
|
<div class="mb-3">
|
||||||
|
Reference in New Issue
Block a user