From 701a9a552e95f7118cbc1e1d5eea0e4bad7d7829 Mon Sep 17 00:00:00 2001 From: YoVinchen Date: Mon, 4 Aug 2025 22:07:42 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dredis=E8=AF=86=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/redis_query.py | 134 +++++++++++++++++-------------------- static/js/redis_compare.js | 37 ++++++---- 2 files changed, 87 insertions(+), 84 deletions(-) diff --git a/modules/redis_query.py b/modules/redis_query.py index 69bc321..c73ab12 100644 --- a/modules/redis_query.py +++ b/modules/redis_query.py @@ -86,7 +86,7 @@ def get_random_keys_from_redis(redis_client, count=100, pattern="*", performance def get_redis_values_by_keys(redis_client, keys, cluster_name="Redis集群", performance_tracker=None): """ - 批量查询Redis中指定keys的值，自动适配单节点和集群模式 + 批量查询Redis中指定keys的值，支持所有Redis数据类型（String、Hash、List、Set、ZSet等） Args: redis_client: Redis客户端 @@ -95,50 +95,39 @@ def get_redis_values_by_keys(redis_client, keys, cluster_name="Redis集群", per performance_tracker: 性能追踪器 Returns: - list: 对应keys的值列表，如果key不存在则为None + list: 对应keys的值信息字典列表，包含类型、值和显示格式 """ - start_time = time.time() - result = [None] * len(keys) + from .redis_types import get_redis_value_with_type - logger.info(f"开始从{cluster_name}批量查询 {len(keys)} 个keys") - query_log_collector.add_log('INFO', f"开始从{cluster_name}批量查询 {len(keys)} 个keys") + start_time = time.time() + result = [] + + logger.info(f"开始从{cluster_name}批量查询 {len(keys)} 个keys（支持所有数据类型）") + query_log_collector.add_log('INFO', f"开始从{cluster_name}批量查询 {len(keys)} 个keys（支持所有数据类型）") try: - # 检查是否是集群模式 - is_cluster = hasattr(redis_client, 'cluster_nodes') - - if is_cluster: - # 集群模式：按slot分组keys以优化查询性能 - slot_groups = {} - for idx, key in enumerate(keys): - slot = key_slot(key) - slot_groups.setdefault(slot, []).append((idx, key)) - - logger.info(f"集群模式：keys分布在 {len(slot_groups)} 个slot中") - query_log_collector.add_log('INFO', f"集群模式：keys分布在 {len(slot_groups)} 个slot中") - - # 分组批量查询 - for group in slot_groups.values(): - indices, slot_keys = zip(*group) - values = redis_client.mget(slot_keys) - for i, v in zip(indices, values): - result[i] = v - else: - # 单节点模式：直接批量查询 - logger.info(f"单节点模式：直接批量查询") - query_log_collector.add_log('INFO', f"单节点模式：直接批量查询") - result = redis_client.mget(keys) + # 逐个查询每个key，支持所有Redis数据类型 + for key in keys: + key_info = get_redis_value_with_type(redis_client, key) + result.append(key_info) end_time = time.time() query_duration = end_time - start_time if performance_tracker: - performance_tracker.record_query(f"{cluster_name}_batch_query", query_duration) + performance_tracker.record_query(f"{cluster_name}_typed_batch_query", query_duration) - # 统计成功获取的key数量 - successful_count = sum(1 for v in result if v is not None) - logger.info(f"从{cluster_name}查询完成，成功获取 {successful_count}/{len(keys)} 个值，耗时 {query_duration:.3f} 秒") - query_log_collector.add_log('INFO', f"从{cluster_name}查询完成，成功获取 {successful_count}/{len(keys)} 个值，耗时 {query_duration:.3f} 秒") + # 统计成功获取的key数量和类型分布 + successful_count = sum(1 for r in result if r['exists']) + type_stats = {} + for r in result: + if r['exists']: + key_type = r['type'] + type_stats[key_type] = type_stats.get(key_type, 0) + 1 + + type_info = ", ".join([f"{t}: {c}" for t, c in type_stats.items()]) if type_stats else "无" + logger.info(f"从{cluster_name}查询完成，成功获取 {successful_count}/{len(keys)} 个值，数据类型分布: [{type_info}]，耗时 {query_duration:.3f} 秒") + query_log_collector.add_log('INFO', f"从{cluster_name}查询完成，成功获取 {successful_count}/{len(keys)} 个值，数据类型分布: [{type_info}]，耗时 {query_duration:.3f} 秒") return result @@ -147,15 +136,16 @@ def get_redis_values_by_keys(redis_client, keys, cluster_name="Redis集群", per query_duration = end_time - start_time if performance_tracker: - performance_tracker.record_query(f"{cluster_name}_batch_query_error", query_duration) + performance_tracker.record_query(f"{cluster_name}_typed_batch_query_error", query_duration) logger.error(f"从{cluster_name}批量查询失败: {e}，耗时 {query_duration:.3f} 秒") query_log_collector.add_log('ERROR', f"从{cluster_name}批量查询失败: {e}，耗时 {query_duration:.3f} 秒") - return result + # 返回错误占位符 + return [{'type': 'error', 'value': None, 'display_value': f'', 'exists': False} for _ in keys] def compare_redis_data(client1, client2, keys, cluster1_name="生产集群", cluster2_name="测试集群", performance_tracker=None): """ - 比较两个Redis集群中指定keys的数据 + 比较两个Redis集群中指定keys的数据，支持所有Redis数据类型 Args: client1: 第一个Redis客户端（生产） @@ -168,18 +158,20 @@ def compare_redis_data(client1, client2, keys, cluster1_name="生产集群", clu Returns: dict: 比较结果，包含统计信息和差异详情 """ + from .redis_types import compare_redis_values + comparison_start_time = time.time() - logger.info(f"开始比较 {cluster1_name} 和 {cluster2_name} 的数据") - query_log_collector.add_log('INFO', f"开始比较 {cluster1_name} 和 {cluster2_name} 的数据") + logger.info(f"开始比较 {cluster1_name} 和 {cluster2_name} 的数据（支持所有Redis数据类型）") + query_log_collector.add_log('INFO', f"开始比较 {cluster1_name} 和 {cluster2_name} 的数据（支持所有Redis数据类型）") # 获取两个集群的数据 values1 = get_redis_values_by_keys(client1, keys, cluster1_name, performance_tracker) - if values1 is None: + if not values1: return {'error': f'从{cluster1_name}获取数据失败'} values2 = get_redis_values_by_keys(client2, keys, cluster2_name, performance_tracker) - if values2 is None: + if not values2: return {'error': f'从{cluster2_name}获取数据失败'} # 开始数据比对 @@ -202,56 +194,56 @@ def compare_redis_data(client1, client2, keys, cluster1_name="生产集群", clu # 逐个比较 for i, key in enumerate(keys): - val1 = values1[i] - val2 = values2[i] + key_str = key.decode('utf-8') if isinstance(key, bytes) else key + value1_info = values1[i] + value2_info = values2[i] - # 将bytes转换为字符串用于显示（如果是bytes类型） - display_val1 = val1.decode('utf-8') if isinstance(val1, bytes) else val1 - display_val2 = val2.decode('utf-8') if isinstance(val2, bytes) else val2 + # 使用redis_types模块的比较函数 + comparison_result = compare_redis_values(value1_info, value2_info) - if val1 is None and val2 is None: - # 两个集群都没有这个key + if comparison_result['status'] == 'both_missing': stats['both_missing'] += 1 missing_results.append({ - 'key': key.decode('utf-8') if isinstance(key, bytes) else key, + 'key': key_str, 'status': 'both_missing', - 'message': '两个集群都不存在该key' + 'message': comparison_result['message'] }) - elif val1 is None: - # 只有第一个集群没有 + elif comparison_result['status'] == 'missing_in_cluster1': stats['missing_in_cluster1'] += 1 missing_results.append({ - 'key': key.decode('utf-8') if isinstance(key, bytes) else key, + 'key': key_str, 'status': 'missing_in_cluster1', 'cluster1_value': None, - 'cluster2_value': display_val2, - 'message': f'在{cluster1_name}中不存在' + 'cluster2_value': value2_info['display_value'], + 'cluster2_type': value2_info['type'], + 'message': comparison_result['message'] }) - elif val2 is None: - # 只有第二个集群没有 + elif comparison_result['status'] == 'missing_in_cluster2': stats['missing_in_cluster2'] += 1 missing_results.append({ - 'key': key.decode('utf-8') if isinstance(key, bytes) else key, + 'key': key_str, 'status': 'missing_in_cluster2', - 'cluster1_value': display_val1, + 'cluster1_value': value1_info['display_value'], + 'cluster1_type': value1_info['type'], 'cluster2_value': None, - 'message': f'在{cluster2_name}中不存在' + 'message': comparison_result['message'] }) - elif val1 == val2: - # 值相同 + elif comparison_result['status'] == 'identical': stats['identical_count'] += 1 identical_results.append({ - 'key': key.decode('utf-8') if isinstance(key, bytes) else key, - 'value': display_val1 + 'key': key_str, + 'value': value1_info['display_value'], + 'type': value1_info['type'] }) - else: - # 值不同 + else: # different stats['different_count'] += 1 different_results.append({ - 'key': key.decode('utf-8') if isinstance(key, bytes) else key, - 'cluster1_value': display_val1, - 'cluster2_value': display_val2, - 'message': '值不同' + 'key': key_str, + 'cluster1_value': value1_info['display_value'], + 'cluster1_type': value1_info['type'], + 'cluster2_value': value2_info['display_value'], + 'cluster2_type': value2_info['type'], + 'message': comparison_result['message'] }) compare_end = time.time() diff --git a/static/js/redis_compare.js b/static/js/redis_compare.js index 6da4364..8a758f5 100644 --- a/static/js/redis_compare.js +++ b/static/js/redis_compare.js @@ -399,16 +399,19 @@ function displayDifferenceResults(differences) { let html = ''; differences.forEach((diff, index) => { + const cluster1Type = diff.cluster1_type ? ` (${diff.cluster1_type})` : ''; + const cluster2Type = diff.cluster2_type ? ` (${diff.cluster2_type})` : ''; + html += `
Key: ${diff.key}
- ${currentResults.clusters.cluster1_name || '集群1'}: + ${currentResults.clusters.cluster1_name || '集群1'}${cluster1Type}: 
${formatRedisValue(diff.cluster1_value)}
- ${currentResults.clusters.cluster2_name || '集群2'}: + ${currentResults.clusters.cluster2_name || '集群2'}${cluster2Type}: 
${formatRedisValue(diff.cluster2_value)}
@@ -439,11 +442,13 @@ function displayIdenticalResults(identical) { let html = ''; identical.forEach((item, index) => { + const typeInfo = item.type ? ` (${item.type})` : ''; + html += `
Key: ${item.key}
- 值: + 值${typeInfo}: 
${formatRedisValue(item.value)}
@@ -479,15 +484,15 @@ function displayMissingResults(missing) {
${item.message}
- ${item.cluster1_value !== undefined ? ` + ${item.cluster1_value !== undefined && item.cluster1_value !== null ? `
- ${currentResults.clusters.cluster1_name || '集群1'}: + ${currentResults.clusters.cluster1_name || '集群1'}${item.cluster1_type ? ` (${item.cluster1_type})` : ''}: 
${formatRedisValue(item.cluster1_value)}
` : ''} - ${item.cluster2_value !== undefined ? ` + ${item.cluster2_value !== undefined && item.cluster2_value !== null ? `
- ${currentResults.clusters.cluster2_name || '集群2'}: + ${currentResults.clusters.cluster2_name || '集群2'}${item.cluster2_type ? ` (${item.cluster2_type})` : ''}: 
${formatRedisValue(item.cluster2_value)}
` : ''} @@ -1674,6 +1679,7 @@ function displayRawData(results) { function collectClusterData(results, clusterType) { const data = []; const clusterField = clusterType === 'cluster1' ? 'cluster1_value' : 'cluster2_value'; + const clusterTypeField = clusterType === 'cluster1' ? 'cluster1_type' : 'cluster2_type'; // 从相同结果中收集数据 if (results.identical_results) { @@ -1682,7 +1688,8 @@ function collectClusterData(results, clusterType) { data.push({ key: item.key, value: item.value, - type: 'identical' + type: 'identical', + redis_type: item.type || 'unknown' }); } }); @@ -1695,7 +1702,8 @@ function collectClusterData(results, clusterType) { data.push({ key: item.key, value: item[clusterField], - type: 'different' + type: 'different', + redis_type: item[clusterTypeField] || 'unknown' }); } }); @@ -1704,11 +1712,12 @@ function collectClusterData(results, clusterType) { // 从缺失结果中收集数据 if (results.missing_results) { results.missing_results.forEach(item => { - if (item.key && item[clusterField] !== undefined) { + if (item.key && item[clusterField] !== undefined && item[clusterField] !== null) { data.push({ key: item.key, value: item[clusterField], - type: 'missing' + type: 'missing', + redis_type: item[clusterTypeField] || 'unknown' }); } }); @@ -1740,11 +1749,12 @@ function renderRawData(clusterId, data, viewType) { data.forEach((item, index) => { const typeClass = getTypeClass(item.type); const formattedValue = formatRedisValue(item.value); + const redisTypeInfo = item.redis_type ? ` [${item.redis_type}]` : ''; html += `
- Key: ${escapeHtml(item.key)} + Key: ${escapeHtml(item.key)}${redisTypeInfo} ${item.type}
@@ -1760,7 +1770,8 @@ function renderRawData(clusterId, data, viewType) { html += '
';
         data.forEach((item, index) => {
             html += `Key: ${escapeHtml(item.key)}\n`;
-            html += `Type: ${item.type}\n`;
+            html += `Redis Type: ${item.redis_type || 'unknown'}\n`;
+            html += `Comparison Type: ${item.type}\n`;
             html += `Value: ${escapeHtml(String(item.value))}\n`;
             if (index < data.length - 1) {
                 html += '\n' + '='.repeat(50) + '\n\n';