commit 6fecd70ca5647258020b01bdb46d0dfbe49098fe Author: YoVinchen Date: Thu Jul 31 18:05:10 2025 +0800 初始化项目 diff --git a/app.py b/app.py new file mode 100644 index 0000000..5b69a59 --- /dev/null +++ b/app.py @@ -0,0 +1,754 @@ +from flask import Flask, render_template, request, jsonify +from cassandra.cluster import Cluster +from cassandra.auth import PlainTextAuthProvider +import json +import os +import logging +import sqlite3 +from datetime import datetime + +app = Flask(__name__) + +# 配置日志 +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# 数据库配置 +DATABASE_PATH = 'config_groups.db' + +def init_database(): + """初始化数据库""" + try: + conn = sqlite3.connect(DATABASE_PATH) + cursor = conn.cursor() + + # 创建配置组表 + cursor.execute(''' + CREATE TABLE IF NOT EXISTS config_groups ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + description TEXT, + pro_config TEXT NOT NULL, + test_config TEXT NOT NULL, + query_config TEXT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + ''') + + conn.commit() + conn.close() + logger.info("数据库初始化完成") + return True + except Exception as e: + logger.error(f"数据库初始化失败: {e}") + return False + +def ensure_database(): + """确保数据库和表存在""" + if not os.path.exists(DATABASE_PATH): + logger.info("数据库文件不存在，正在创建...") + return init_database() + + # 检查表是否存在 + try: + conn = sqlite3.connect(DATABASE_PATH) + cursor = conn.cursor() + cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='config_groups'") + result = cursor.fetchone() + conn.close() + + if not result: + logger.info("config_groups表不存在，正在创建...") + return init_database() + + return True + except Exception as e: + logger.error(f"检查数据库表失败: {e}") + return init_database() + +def get_db_connection(): + """获取数据库连接""" + conn = sqlite3.connect(DATABASE_PATH) + conn.row_factory = sqlite3.Row + return conn + +def normalize_json_string(value): + """标准化JSON字符串，用于比较""" + if not isinstance(value, str): + return value + + try: + # 尝试解析JSON + json_obj = json.loads(value) + + # 如果是数组，需要进行特殊处理 + if isinstance(json_obj, list): + # 尝试对数组元素进行标准化排序 + normalized_array = normalize_json_array(json_obj) + return json.dumps(normalized_array, sort_keys=True, separators=(',', ':')) + else: + # 普通对象，直接序列化 + return json.dumps(json_obj, sort_keys=True, separators=(',', ':')) + except (json.JSONDecodeError, TypeError): + # 如果不是JSON，返回原值 + return value + +def normalize_json_array(json_array): + """标准化JSON数组，处理元素顺序问题""" + try: + normalized_elements = [] + + for element in json_array: + if isinstance(element, dict): + # 对字典元素进行标准化 + normalized_elements.append(json.dumps(element, sort_keys=True, separators=(',', ':'))) + elif isinstance(element, str): + # 如果是字符串，尝试解析为JSON + try: + parsed_element = json.loads(element) + normalized_elements.append(json.dumps(parsed_element, sort_keys=True, separators=(',', ':'))) + except: + normalized_elements.append(element) + else: + normalized_elements.append(element) + + # 对标准化后的元素进行排序，确保顺序一致 + normalized_elements.sort() + + # 重新解析为对象数组 + result_array = [] + for element in normalized_elements: + if isinstance(element, str): + try: + result_array.append(json.loads(element)) + except: + result_array.append(element) + else: + result_array.append(element) + + return result_array + + except Exception as e: + logger.warning(f"数组标准化失败: {e}") + return json_array + +def is_json_array_field(value): + """检查字段是否为JSON数组格式""" + if not isinstance(value, (str, list)): + return False + + try: + if isinstance(value, str): + parsed = json.loads(value) + return isinstance(parsed, list) + elif isinstance(value, list): + # 检查是否为JSON字符串数组 + if len(value) > 0 and isinstance(value[0], str): + try: + json.loads(value[0]) + return True + except: + return False + return True + except: + return False + +def compare_array_values(value1, value2): + """专门用于比较数组类型的值""" + try: + # 处理字符串表示的数组 + if isinstance(value1, str) and isinstance(value2, str): + try: + array1 = json.loads(value1) + array2 = json.loads(value2) + if isinstance(array1, list) and isinstance(array2, list): + return compare_json_arrays(array1, array2) + except: + pass + + # 处理Python列表类型 + elif isinstance(value1, list) and isinstance(value2, list): + return compare_json_arrays(value1, value2) + + # 处理混合情况：一个是字符串数组，一个是列表 + elif isinstance(value1, list) and isinstance(value2, str): + try: + array2 = json.loads(value2) + if isinstance(array2, list): + return compare_json_arrays(value1, array2) + except: + pass + elif isinstance(value1, str) and isinstance(value2, list): + try: + array1 = json.loads(value1) + if isinstance(array1, list): + return compare_json_arrays(array1, value2) + except: + pass + + return False + except Exception as e: + logger.warning(f"数组比较失败: {e}") + return False + +def compare_json_arrays(array1, array2): + """比较两个JSON数组，忽略元素顺序""" + try: + if len(array1) != len(array2): + return False + + # 标准化两个数组 + normalized_array1 = normalize_json_array(array1.copy()) + normalized_array2 = normalize_json_array(array2.copy()) + + # 将标准化后的数组转换为可比较的格式 + comparable1 = json.dumps(normalized_array1, sort_keys=True) + comparable2 = json.dumps(normalized_array2, sort_keys=True) + + return comparable1 == comparable2 + + except Exception as e: + logger.warning(f"JSON数组比较失败: {e}") + return False + +def format_json_for_display(value): + """格式化JSON用于显示""" + if not isinstance(value, str): + return str(value) + + try: + # 尝试解析JSON + json_obj = json.loads(value) + # 格式化显示（带缩进） + return json.dumps(json_obj, sort_keys=True, indent=2, ensure_ascii=False) + except (json.JSONDecodeError, TypeError): + # 如果不是JSON，返回原值 + return str(value) + +def is_json_field(value): + """检查字段是否为JSON格式""" + if not isinstance(value, str): + return False + + try: + json.loads(value) + return True + except (json.JSONDecodeError, TypeError): + return False + +def compare_values(value1, value2): + """智能比较两个值，支持JSON标准化和数组比较""" + # 首先检查是否为数组类型 + if is_json_array_field(value1) or is_json_array_field(value2): + return compare_array_values(value1, value2) + + # 如果两个值都是字符串，尝试JSON标准化比较 + if isinstance(value1, str) and isinstance(value2, str): + normalized_value1 = normalize_json_string(value1) + normalized_value2 = normalize_json_string(value2) + return normalized_value1 == normalized_value2 + + # 其他情况直接比较 + return value1 == value2 + +# 默认配置（不显示敏感信息） +DEFAULT_CONFIG = { + 'pro_config': { + 'cluster_name': '', + 'hosts': [], + 'port': 9042, + 'datacenter': '', + 'username': '', + 'password': '', + 'keyspace': '', + 'table': '' + }, + 'test_config': { + 'cluster_name': '', + 'hosts': [], + 'port': 9042, + 'datacenter': '', + 'username': '', + 'password': '', + 'keyspace': '', + 'table': '' + }, + 'keys': ['docid'], + 'fields_to_compare': [], + 'exclude_fields': [] +} + +def save_config_group(name, description, pro_config, test_config, query_config): + """保存配置组""" + if not ensure_database(): + logger.error("数据库初始化失败") + return False + + conn = get_db_connection() + cursor = conn.cursor() + + try: + cursor.execute(''' + INSERT OR REPLACE INTO config_groups + (name, description, pro_config, test_config, query_config, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ''', ( + name, description, + json.dumps(pro_config), + json.dumps(test_config), + json.dumps(query_config), + datetime.now().isoformat() + )) + conn.commit() + logger.info(f"配置组 '{name}' 保存成功") + return True + except Exception as e: + logger.error(f"保存配置组失败: {e}") + return False + finally: + conn.close() + +def get_config_groups(): + """获取所有配置组""" + if not ensure_database(): + logger.error("数据库初始化失败") + return [] + + conn = get_db_connection() + cursor = conn.cursor() + + try: + cursor.execute(''' + SELECT id, name, description, created_at, updated_at + FROM config_groups + ORDER BY updated_at DESC + ''') + rows = cursor.fetchall() + + config_groups = [] + for row in rows: + config_groups.append({ + 'id': row['id'], + 'name': row['name'], + 'description': row['description'], + 'created_at': row['created_at'], + 'updated_at': row['updated_at'] + }) + + return config_groups + except Exception as e: + logger.error(f"获取配置组失败: {e}") + return [] + finally: + conn.close() + +def get_config_group_by_id(group_id): + """根据ID获取配置组详情""" + if not ensure_database(): + logger.error("数据库初始化失败") + return None + + conn = get_db_connection() + cursor = conn.cursor() + + try: + cursor.execute(''' + SELECT * FROM config_groups WHERE id = ? + ''', (group_id,)) + row = cursor.fetchone() + + if row: + return { + 'id': row['id'], + 'name': row['name'], + 'description': row['description'], + 'pro_config': json.loads(row['pro_config']), + 'test_config': json.loads(row['test_config']), + 'query_config': json.loads(row['query_config']), + 'created_at': row['created_at'], + 'updated_at': row['updated_at'] + } + return None + except Exception as e: + logger.error(f"获取配置组详情失败: {e}") + return None + finally: + conn.close() + +def delete_config_group(group_id): + """删除配置组""" + if not ensure_database(): + logger.error("数据库初始化失败") + return False + + conn = get_db_connection() + cursor = conn.cursor() + + try: + cursor.execute('DELETE FROM config_groups WHERE id = ?', (group_id,)) + conn.commit() + success = cursor.rowcount > 0 + if success: + logger.info(f"配置组ID {group_id} 删除成功") + return success + except Exception as e: + logger.error(f"删除配置组失败: {e}") + return False + finally: + conn.close() + +def create_connection(config): + """创建Cassandra连接""" + try: + auth_provider = PlainTextAuthProvider(username=config['username'], password=config['password']) + cluster = Cluster(config['hosts'], port=config['port'], auth_provider=auth_provider) + session = cluster.connect(config['keyspace']) + return cluster, session + except Exception as e: + return None, None + +def execute_query(session, table, keys, fields, values, exclude_fields=None): + """执行查询""" + try: + # 构建查询条件 + quoted_values = [f"'{value}'" for value in values] + query_conditions = f"{keys[0]} IN ({', '.join(quoted_values)})" + + # 确定要查询的字段 + if fields: + fields_str = ", ".join(fields) + else: + fields_str = "*" + + query_sql = f"SELECT {fields_str} FROM {table} WHERE {query_conditions};" + result = session.execute(query_sql) + return list(result) if result else [] + except Exception as e: + return [] + +def compare_results(pro_data, test_data, keys, fields_to_compare, exclude_fields, values): + """比较查询结果""" + differences = [] + field_diff_count = {} + identical_results = [] # 存储相同的结果 + + for value in values: + # 查找原表和测试表中该ID的相关数据 + rows_pro = [row for row in pro_data if getattr(row, keys[0]) == value] + rows_test = [row for row in test_data if getattr(row, keys[0]) == value] + + for row_pro in rows_pro: + # 在测试表中查找相同主键的行 + row_test = next( + (row for row in rows_test if all(getattr(row, key) == getattr(row_pro, key) for key in keys)), + None + ) + + if row_test: + # 确定要比较的列 + columns = fields_to_compare if fields_to_compare else row_pro._fields + columns = [col for col in columns if col not in exclude_fields] + + has_difference = False + row_differences = [] + identical_fields = {} + + for column in columns: + value_pro = getattr(row_pro, column) + value_test = getattr(row_test, column) + + # 使用智能比较函数 + if not compare_values(value_pro, value_test): + has_difference = True + # 格式化显示值 + formatted_pro_value = format_json_for_display(value_pro) + formatted_test_value = format_json_for_display(value_test) + + row_differences.append({ + 'key': {key: getattr(row_pro, key) for key in keys}, + 'field': column, + 'pro_value': formatted_pro_value, + 'test_value': formatted_test_value, + 'is_json': is_json_field(value_pro) or is_json_field(value_test), + 'is_array': is_json_array_field(value_pro) or is_json_array_field(value_test) + }) + + # 统计字段差异次数 + field_diff_count[column] = field_diff_count.get(column, 0) + 1 + else: + # 存储相同的字段值 + identical_fields[column] = format_json_for_display(value_pro) + + if has_difference: + differences.extend(row_differences) + else: + # 如果没有差异，存储到相同结果中 + identical_results.append({ + 'key': {key: getattr(row_pro, key) for key in keys}, + 'pro_fields': identical_fields, + 'test_fields': {col: format_json_for_display(getattr(row_test, col)) for col in columns} + }) + else: + # 在测试表中未找到对应行 + differences.append({ + 'key': {key: getattr(row_pro, key) for key in keys}, + 'message': '在测试表中未找到该行' + }) + + # 检查测试表中是否有生产表中不存在的行 + for row_test in rows_test: + row_pro = next( + (row for row in rows_pro if all(getattr(row, key) == getattr(row_test, key) for key in keys)), + None + ) + if not row_pro: + differences.append({ + 'key': {key: getattr(row_test, key) for key in keys}, + 'message': '在生产表中未找到该行' + }) + + return differences, field_diff_count, identical_results + +def generate_comparison_summary(total_keys, pro_count, test_count, differences, identical_results, field_diff_count): + """生成比较总结报告""" + # 计算基本统计 + different_records = len(set([list(diff['key'].values())[0] for diff in differences if 'field' in diff])) + identical_records = len(identical_results) + missing_in_test = len([diff for diff in differences if diff.get('message') == '在测试表中未找到该行']) + missing_in_pro = len([diff for diff in differences if diff.get('message') == '在生产表中未找到该行']) + + # 计算百分比 + def safe_percentage(part, total): + return round((part / total * 100), 2) if total > 0 else 0 + + identical_percentage = safe_percentage(identical_records, total_keys) + different_percentage = safe_percentage(different_records, total_keys) + + # 生成总结 + summary = { + 'overview': { + 'total_keys_queried': total_keys, + 'pro_records_found': pro_count, + 'test_records_found': test_count, + 'identical_records': identical_records, + 'different_records': different_records, + 'missing_in_test': missing_in_test, + 'missing_in_pro': missing_in_pro + }, + 'percentages': { + 'data_consistency': identical_percentage, + 'data_differences': different_percentage, + 'missing_rate': safe_percentage(missing_in_test + missing_in_pro, total_keys) + }, + 'field_analysis': { + 'total_fields_compared': len(field_diff_count) if field_diff_count else 0, + 'most_different_fields': sorted(field_diff_count.items(), key=lambda x: x[1], reverse=True)[:5] if field_diff_count else [] + }, + 'data_quality': { + 'completeness': safe_percentage(pro_count + test_count, total_keys * 2), + 'consistency_score': identical_percentage, + 'quality_level': get_quality_level(identical_percentage) + }, + 'recommendations': generate_recommendations(identical_percentage, missing_in_test, missing_in_pro, field_diff_count) + } + + return summary + +def get_quality_level(consistency_percentage): + """根据一致性百分比获取数据质量等级""" + if consistency_percentage >= 95: + return {'level': '优秀', 'color': 'success', 'description': '数据一致性非常高'} + elif consistency_percentage >= 90: + return {'level': '良好', 'color': 'info', 'description': '数据一致性较高'} + elif consistency_percentage >= 80: + return {'level': '一般', 'color': 'warning', 'description': '数据一致性中等，需要关注'} + else: + return {'level': '较差', 'color': 'danger', 'description': '数据一致性较低，需要重点处理'} + +def generate_recommendations(consistency_percentage, missing_in_test, missing_in_pro, field_diff_count): + """生成改进建议""" + recommendations = [] + + if consistency_percentage < 90: + recommendations.append('建议重点关注数据一致性问题，检查数据同步机制') + + if missing_in_test > 0: + recommendations.append(f'测试环境缺失 {missing_in_test} 条记录，建议检查数据迁移过程') + + if missing_in_pro > 0: + recommendations.append(f'生产环境缺失 {missing_in_pro} 条记录，建议检查数据完整性') + + if field_diff_count: + top_diff_field = max(field_diff_count.items(), key=lambda x: x[1]) + recommendations.append(f'字段 "{top_diff_field[0]}" 差异最多({top_diff_field[1]}次)，建议优先处理') + + if not recommendations: + recommendations.append('数据质量良好，建议继续保持当前的数据管理流程') + + return recommendations + +@app.route('/') +def index(): + return render_template('index.html') + +@app.route('/db-compare') +def db_compare(): + return render_template('db_compare.html') + +@app.route('/api/query', methods=['POST']) +def query_compare(): + try: + data = request.json + logger.info("开始执行数据库比对查询") + + # 解析配置 + pro_config = data.get('pro_config', DEFAULT_CONFIG['pro_config']) + test_config = data.get('test_config', DEFAULT_CONFIG['test_config']) + keys = data.get('keys', DEFAULT_CONFIG['keys']) + fields_to_compare = data.get('fields_to_compare', DEFAULT_CONFIG['fields_to_compare']) + exclude_fields = data.get('exclude_fields', DEFAULT_CONFIG['exclude_fields']) + values = data.get('values', []) + + if not values: + logger.warning("查询失败：未提供查询key值") + return jsonify({'error': '请提供查询key值'}), 400 + + logger.info(f"查询配置：{len(values)}个key值，生产表：{pro_config['table']}，测试表：{test_config['table']}") + + # 创建数据库连接 + pro_cluster, pro_session = create_connection(pro_config) + test_cluster, test_session = create_connection(test_config) + + if not pro_session or not test_session: + logger.error("数据库连接失败") + return jsonify({'error': '数据库连接失败，请检查配置信息'}), 500 + + try: + # 执行查询 + logger.info("执行生产环境查询") + pro_data = execute_query(pro_session, pro_config['table'], keys, fields_to_compare, values, exclude_fields) + logger.info("执行测试环境查询") + test_data = execute_query(test_session, test_config['table'], keys, fields_to_compare, values, exclude_fields) + + logger.info(f"查询结果：生产表 {len(pro_data)} 条记录，测试表 {len(test_data)} 条记录") + + # 比较结果 + differences, field_diff_count, identical_results = compare_results(pro_data, test_data, keys, fields_to_compare, exclude_fields, values) + + # 统计信息 + different_ids = set() + for diff in differences: + if 'field' in diff: + different_ids.add(list(diff['key'].values())[0]) + + non_different_ids = set(values) - different_ids + + # 生成比较总结 + summary = generate_comparison_summary( + len(values), len(pro_data), len(test_data), + differences, identical_results, field_diff_count + ) + + result = { + 'total_keys': len(values), + 'pro_count': len(pro_data), + 'test_count': len(test_data), + 'differences': differences, + 'identical_results': identical_results, + 'field_diff_count': field_diff_count, + 'different_ids': list(different_ids), + 'non_different_ids': list(non_different_ids), + 'summary': summary, + 'raw_pro_data': [dict(row._asdict()) for row in pro_data] if pro_data else [], + 'raw_test_data': [dict(row._asdict()) for row in test_data] if test_data else [] + } + + logger.info(f"比对完成：发现 {len(differences)} 处差异") + return jsonify(result) + + except Exception as e: + logger.error(f"查询执行失败：{str(e)}") + return jsonify({'error': f'查询执行失败：{str(e)}'}), 500 + finally: + # 关闭连接 + if pro_cluster: + pro_cluster.shutdown() + if test_cluster: + test_cluster.shutdown() + + except Exception as e: + logger.error(f"请求处理失败：{str(e)}") + return jsonify({'error': f'请求处理失败：{str(e)}'}), 500 + +@app.route('/api/default-config') +def get_default_config(): + return jsonify(DEFAULT_CONFIG) + +# 配置组管理API +@app.route('/api/config-groups', methods=['GET']) +def api_get_config_groups(): + """获取所有配置组""" + config_groups = get_config_groups() + return jsonify({'success': True, 'data': config_groups}) + +@app.route('/api/config-groups', methods=['POST']) +def api_save_config_group(): + """保存配置组""" + try: + data = request.json + name = data.get('name', '').strip() + description = data.get('description', '').strip() + pro_config = data.get('pro_config', {}) + test_config = data.get('test_config', {}) + query_config = { + 'keys': data.get('keys', []), + 'fields_to_compare': data.get('fields_to_compare', []), + 'exclude_fields': data.get('exclude_fields', []) + } + + if not name: + return jsonify({'success': False, 'error': '配置组名称不能为空'}), 400 + + success = save_config_group(name, description, pro_config, test_config, query_config) + + if success: + return jsonify({'success': True, 'message': '配置组保存成功'}) + else: + return jsonify({'success': False, 'error': '配置组保存失败'}), 500 + + except Exception as e: + logger.error(f"保存配置组API失败: {e}") + return jsonify({'success': False, 'error': str(e)}), 500 + +@app.route('/api/config-groups/', methods=['GET']) +def api_get_config_group(group_id): + """获取指定配置组详情""" + config_group = get_config_group_by_id(group_id) + + if config_group: + return jsonify({'success': True, 'data': config_group}) + else: + return jsonify({'success': False, 'error': '配置组不存在'}), 404 + +@app.route('/api/config-groups/', methods=['DELETE']) +def api_delete_config_group(group_id): + """删除配置组""" + success = delete_config_group(group_id) + + if success: + return jsonify({'success': True, 'message': '配置组删除成功'}) + else: + return jsonify({'success': False, 'error': '配置组删除失败'}), 500 + +@app.route('/api/init-db', methods=['POST']) +def api_init_database(): + """手动初始化数据库（用于测试）""" + success = init_database() + if success: + return jsonify({'success': True, 'message': '数据库初始化成功'}) + else: + return jsonify({'success': False, 'error': '数据库初始化失败'}), 500 + +if __name__ == '__main__': + app.run(debug=True) diff --git a/demo/Query.py b/demo/Query.py new file mode 100644 index 0000000..c63469d --- /dev/null +++ b/demo/Query.py @@ -0,0 +1,276 @@ +from cassandra.cluster import Cluster +from cassandra.auth import PlainTextAuthProvider +from cassandra.cqlengine.columns import Boolean +from cassandra.policies import ColDesc + +# CBase Hot +cluster_nodes_pro = ['10.20.2.22'] # Cassandra节点的IP地址 +port_pro = 9042 # Cassandra使用的端口 +username_pro = 'cbase' # Cassandra用户名 +password_pro = 'antducbaseadmin@2022' # Cassandra密码 +keyspace_pro = 'yuqing_skinny' # Cassandra keyspace_pro + +cluster_nodes_test = ['10.20.2.22'] # Cassandra节点的IP地址 +port_test = 9042 # Cassandra使用的端口 +username_test = 'cbase' # Cassandra用户名 +password_test = 'antducbaseadmin@2022' # Cassandra密码 +keyspace_test = 'yuqing_skinny' # Cassandra keyspace_pro + +# CBase Cold +# cluster_nodes_pro = ['10.20.1.108'] # Cassandra节点的IP地址 +# port_pro = 9042 # Cassandra使用的端口 +# username_pro = 'cassandra' # Cassandra用户名 +# password_pro = 'cassandra' # Cassandra密码 +# keyspace_pro = 'yuqing_skinny' # Cassandra keyspace_pro +# cluster_nodes_test = ['10.20.1.108'] # Cassandra节点的IP地址 +# port_test = 9042 # Cassandra使用的端口 +# username_test = 'cassandra' # Cassandra用户名 +# password_test = 'cassandra' # Cassandra密码 +# keyspace_test = 'yuqing_skinny' # Cassandra + +# cluster_nodes_pro = ['10.20.1.119'] # Cassandra节点的IP地址 +# port_pro = 9044 # Cassandra使用的端口 +# username_pro = 'cbase' # Cassandra用户名 +# password_pro = 'antducbaseadmin@2022' # Cassandra密码 +# keyspace_pro = 'yuqing_skinny' # Cassandra keyspace_pro + +data_table_pro = "document" +data_table_test = data_table_pro + "_test" + +values = [] # 多个ID值 + +# data_table_pro = "doc_view_8" +# data_table_test = "doc_view_test" + +# 定义主键字段及其对应的多ID值 +keys = ["docid"] + +# 比较全部字段 +fields_to_compare = [] +# fields_to_compare = [ +# "statusid", +# "taglocation", +# "tagemotion", +# "tagindustry", +# "tagdomain", +# "tagtopic", +# "tagsimilar", +# "tagother", +# "hasprivacy", +# "istaged", +# "createat", +# ] # 指定要比较的字段 + +exclude_fields = [] # 需要排除的字段 +# exclude_fields = ['mcrelated','ocrtexts',''] + +# 小写转换 +fields_to_compare = [field.lower() for field in fields_to_compare] +exclude_fields = [field.lower() for field in exclude_fields] +keys = [field.lower() for field in keys] + +# 定义存储字段差异数量的字典 +field_diff_count = {} + +# 输出文件 +output_file = "/Users/yovinchen/project/python/CassandraQueryComparator/QueryCassandra/output.txt" +input_file = "/Users/yovinchen/project/python/CassandraQueryComparator/QueryCassandra/input.txt" + +# 清空文件内容 +open(output_file, "w").close() + +with open(input_file, "r") as file: + values = [item.replace('"', '') for line in file for item in line.strip().split(",") if item] + +# 创建身份验证提供程序 +auth_provider = PlainTextAuthProvider(username=username_pro, password=password_pro) +# 连接到Cassandra集群 +cluster = Cluster(cluster_nodes_pro, port=port_pro, auth_provider=auth_provider) +session = cluster.connect(keyspace_pro) # 连接到指定的keyspace + +# 创建身份验证提供程序 +auth_provider1 = PlainTextAuthProvider(username=username_test, password=password_test) +# 连接到Cassandra集群 +cluster1 = Cluster(cluster_nodes_test, port=port_test, auth_provider=auth_provider1) +session1 = cluster1.connect(keyspace_test) # 连接到指定的keyspace +# 构建IN查询语句 +query_conditions = f"""{keys[0]} IN ({', '.join([f"'{value}'" for value in values])})""" + +# 如果 fields_to_compare 不为空，使用其中的字段，否则使用 * +fields_str = ", ".join(fields_to_compare) if fields_to_compare else "*" + +query_sql1 = f"SELECT {fields_str} FROM {data_table_pro} WHERE {query_conditions};" +query_sql2 = f"SELECT {fields_str} FROM {data_table_test} WHERE {query_conditions};" + +# 执行查询 +result_doc_data = session.execute(query_sql1) +result_doc_data_test = session1.execute(query_sql2) + +# 检查查询结果是否为空，并转换查询结果为列表 +list_doc_data = list(result_doc_data) if result_doc_data else [] +list_doc_data_test = list(result_doc_data_test) if result_doc_data_test else [] + +# list_doc_data = list(result_doc_data) if result_doc_data else [] +# list_doc_data_test = list(result_doc_data_test) if result_doc_data_test else [] + +with open(output_file, "a") as f: + f.write(f"查询 {data_table_pro} 内容和 {data_table_test} 内容：\n") + for item1, item2 in zip(list_doc_data, list_doc_data_test): + f.write(f"{item1}\n{item2}\n\n") + +# with open(output_file, "a") as f: +# f.write(f"查询 {data_table_pro} 内容：\n{list_doc_data}\n") +# f.write(f"查询 {data_table_test} 内容：\n{list_doc_data_test}\n") + +if not list_doc_data: + with open(output_file, "a") as f: + f.write(f"查询 {data_table_pro} 的结果为空。") + print(f"查询 {data_table_pro} 的结果为空。") +if not list_doc_data_test: + with open(output_file, "a") as f: + f.write(f"查询 {data_table_test} 的结果为空。") + print(f"查询 {data_table_test} 的结果为空。") + +# 创建一个列表来存储详细比较结果 +differences = [] + + +# 进行详细比较 +def compare_data(fields_to_compare=None, exclude_fields=None): + exclude_fields = exclude_fields or [] # 如果未指定排除字段，默认为空列表 + + for value in values: + # 查找原表和测试表中该ID的相关数据 + rows_data = [row for row in list_doc_data if getattr(row, keys[0]) == value] + rows_test = [row for row in list_doc_data_test if getattr(row, keys[0]) == value] + + for row_data in rows_data: + # 在 doc_data_test 中查找相同主键的行 + row_test = next( + (row for row in rows_test if all(getattr(row, key) == getattr(row_data, key) for key in keys)), + None) + + if row_test: + # 如果在 doc_data_test 中找到相同的主键，则逐列比较 + columns = fields_to_compare if fields_to_compare else row_data._fields + columns = [col for col in columns if col not in exclude_fields] # 过滤排除字段 + + for column in columns: + value_data = getattr(row_data, column) + value_test = getattr(row_test, column) + if value_data != value_test: + differences.append({ + '主键': {key: getattr(row_data, key) for key in keys}, + '字段': column, + '生产表': f"\n{value_data}\n", + '测试表': f"\n{value_test}\n" + }) + + # 统计字段差异次数 + if column in field_diff_count: + field_diff_count[column] += 1 + else: + field_diff_count[column] = 1 + + else: + # 如果在 doc_data_test 中未找到相同的行 + differences.append({ + '主键': {key: getattr(row_data, key) for key in keys}, + '消息': f'在 {data_table_test} 中未找到该行' + }) + + # 比较 doc_data_test 中的行是否在 doc_data 中存在 + for row_test in rows_test: + row_data = next( + (row for row in rows_data if all(getattr(row, key) == getattr(row_test, key) for key in keys)), None) + if not row_data: + differences.append({ + '主键': {key: getattr(row_test, key) for key in keys}, + '消息': f'在 {data_table_pro} 中未找到该行' + }) + + +compare_data(fields_to_compare, exclude_fields) + +# 使用集合来保存唯一的 topicid +id_set = set() +field_set = set() + +grouped_id_dict = {} + +# 输出指定字段的差异 +with open(output_file, "a") as f: + if differences: + f.write("\n发现指定字段的差异:\n") + for diff in differences: + # 逐行打印每个差异 + f.write(f"主键: {diff['主键']}\n") + f.write(f"字段: {diff.get('字段', 'N/A')}\n") + f.write(f"生产表: {diff.get('生产表', 'N/A')}\n") + f.write(f"测试表: {diff.get('测试表', 'N/A')}\n") + f.write("-" * 50) # 分隔符，便于查看 + f.write("\n") + + # 将差异ID按字段名分组 + field = diff.get('字段', '未分组') + if field not in grouped_id_dict: + grouped_id_dict[field] = set() + for key in keys: + id = diff['主键'][key] + id_set.add('"' + id + '",') + grouped_id_dict[field].add('"' + id + '",') + + # 输出分组后的差异ID + if grouped_id_dict: + f.write("\n差异ID按字段分组如下:") + for field, ids in grouped_id_dict.items(): + field_set.add('"' + field + '",') + f.write(f"字段: {field}\n") + f.write("差异ID:") + for id in ids: + f.write(id) + f.write("\n") + f.write("-" * 50) # 分隔符，便于查看 + f.write("\n") + else: + f.write("\n指定字段未发现差异。\n") + + f.write("\n") + + # 只有在 field_set 不为空时才打印 + if field_set: + f.write("\n存在差异的 字段 为:\n") + # 打印所有唯一的 field + for field in field_set: + f.write(field + "\n") + + # 只有在 id_set 不为空时才打印 + if id_set: + f.write("\n存在差异的 ID 为:\n") + # 打印所有唯一的 topicid + for topicid in id_set: + f.write(topicid + "\n") + f.write("\n") + + # 计算存在差异的 ID + different_ids = {id.strip('"').strip(',').strip('"') for id in id_set} + + # 计算不存在差异的 ID（即在 values 中但不在 different_ids 中） + non_different_ids = set(values) - different_ids + + # 只有在 non_different_ids 非空时才打印 + if non_different_ids: + f.write("\n不存在差异的 ID 为:\n") + for topicid in non_different_ids: + f.write(f'"{topicid}",\n') + f.write("\n") + + f.write("总计key " + len(values).__str__() + "个") + # 统计每个字段的差异数量 + if field_diff_count: + f.write("\n字段差异统计如下:\n") + for field, count in field_diff_count.items(): + f.write(f"字段 '{field}' 发现 {count} 处差异\n") + +# 关闭连接 +cluster.shutdown() diff --git a/demo/twcsQuery.py b/demo/twcsQuery.py new file mode 100644 index 0000000..c260401 --- /dev/null +++ b/demo/twcsQuery.py @@ -0,0 +1,283 @@ +from cassandra.cluster import Cluster +from cassandra.auth import PlainTextAuthProvider +from cassandra.cqlengine.columns import Boolean +from cassandra.policies import ColDesc + +# # 配置Cassandra集群信息 +# doc_view +# cluster_nodes_pro = ['10.20.2.43'] # Cassandra节点的IP地址 +# port_pro = 9042 # Cassandra使用的端口 +# username_pro = 'cbase' # Cassandra用户名 +# password_pro = 'antducbaseadmin@2022' # Cassandra密码 +# keyspace_pro = 'yuqing_twcs' # Cassandra keyspace_pro + +# CBase Hot +cluster_nodes_test = ['10.20.2.22'] # Cassandra节点的IP地址 +port_test = 9042 # Cassandra使用的端口 +username_test = 'cbase' # Cassandra用户名 +password_test = 'antducbaseadmin@2022' # Cassandra密码 +keyspace_test = 'yuqing_skinny' # Cassandra keyspace_pro + +# CBase Cold +cluster_nodes_pro = ['10.20.4.152'] # Cassandra节点的IP地址 +port_pro = 9044 # Cassandra使用的端口 +username_pro = 'cbase' # Cassandra用户名 +password_pro = 'antducbaseadmin@2022' # Cassandra密码 +keyspace_pro = 'yuqing_skinny' # Cassandra keyspace_pro + +# cluster_nodes_test = ['10.20.1.108'] # Cassandra节点的IP地址 +# port_test = 9042 # Cassandra使用的端口 +# username_test = 'cassandra' # Cassandra用户名 +# password_test = 'cassandra' # Cassandra密码 +# keyspace_test = 'yuqing_skinny' # Cassandra keyspace_pro + +# cluster_nodes_pro = ['10.20.1.119'] # Cassandra节点的IP地址 +# port_pro = 9044 # Cassandra使用的端口 +# username_pro = 'cbase' # Cassandra用户名 +# password_pro = 'antducbaseadmin@2022' # Cassandra密码 +# keyspace_pro = 'yuqing_skinny' # Cassandra keyspace_pro + + +# data_table_pro = "doc_view" +# data_table_test = data_table_pro + "_test" + +values = [] # 多个ID值 + +data_table_pro = "wemedia_1" +data_table_test = "wemedia_test" + +# 定义主键字段及其对应的多ID值 +keys = ["wmid"] + +# 比较全部字段 +fields_to_compare = [] +# fields_to_compare = [ +# "docid", +# "bitset", +# "crawltime", +# "createat", +# "domain", +# "fanslevel", +# "nickname", +# "officialsitetypes", +# "platform", +# "tagemotion", +# "taglocation", +# "tagsimilar", +# "userid", +# "username", +# ] # 指定要比较的字段 + +exclude_fields = [] # 需要排除的字段 +# exclude_fields = ['mcrelated','ocrtexts',''] + +# 定义存储字段差异数量的字典 +field_diff_count = {} + +# 输出文件 +output_file = "/Users/yovinchen/project/python/CassandraQueryComparator/QueryCassandra/output.txt" +input_file = "/Users/yovinchen/project/python/CassandraQueryComparator/QueryCassandra/input.txt" + +# 清空文件内容 +open(output_file, "w").close() + +# 单表 +# with open(input_file, "r") as file: +# values = [line.strip() for line in file if line.strip()] # 去除空行 + +# twcs +with open(input_file, "r") as file: + values = [item.replace('"', '') for line in file for item in line.strip().split(",") if item] + +# 创建身份验证提供程序 +auth_provider = PlainTextAuthProvider(username=username_pro, password=password_pro) +# 连接到Cassandra集群 +cluster = Cluster(cluster_nodes_pro, port=port_pro, auth_provider=auth_provider) +session = cluster.connect(keyspace_pro) # 连接到指定的keyspace + +# 创建身份验证提供程序 +auth_provider1 = PlainTextAuthProvider(username=username_test, password=password_test) +# 连接到Cassandra集群 +cluster1 = Cluster(cluster_nodes_test, port=port_test, auth_provider=auth_provider1) +session1 = cluster1.connect(keyspace_test) # 连接到指定的keyspace +# 构建IN查询语句 +query_conditions = f"""{keys[0]} IN ({', '.join([f"'{value}'" for value in values])})""" + +# 如果 fields_to_compare 不为空，使用其中的字段，否则使用 * +fields_str = ", ".join(fields_to_compare) if fields_to_compare else "*" + +query_sql1 = f"SELECT {fields_str} FROM {data_table_pro} WHERE {query_conditions};" +query_sql2 = f"SELECT {fields_str} FROM {data_table_test} WHERE {query_conditions};" + +# 执行查询 +result_doc_data = session.execute(query_sql1) +result_doc_data_test = session1.execute(query_sql2) + +# 检查查询结果是否为空，并转换查询结果为列表 +list_doc_data = list(result_doc_data) if result_doc_data else [] +list_doc_data_test = list(result_doc_data_test) if result_doc_data_test else [] + +# list_doc_data = list(result_doc_data) if result_doc_data else [] +# list_doc_data_test = list(result_doc_data_test) if result_doc_data_test else [] + +with open(output_file, "a") as f: + f.write(f"查询 {data_table_pro} 内容和 {data_table_test} 内容：\n") + for item1, item2 in zip(list_doc_data, list_doc_data_test): + f.write(f"{item1}\n{item2}\n\n") + +# with open(output_file, "a") as f: +# f.write(f"查询 {data_table_pro} 内容：\n{list_doc_data}\n") +# f.write(f"查询 {data_table_test} 内容：\n{list_doc_data_test}\n") + +if not list_doc_data: + with open(output_file, "a") as f: + f.write(f"查询 {data_table_pro} 的结果为空。") + print(f"查询 {data_table_pro} 的结果为空。") +if not list_doc_data_test: + with open(output_file, "a") as f: + f.write(f"查询 {data_table_test} 的结果为空。") + print(f"查询 {data_table_test} 的结果为空。") + +# 创建一个列表来存储详细比较结果 +differences = [] + + +# 进行详细比较 +def compare_data(fields_to_compare=None, exclude_fields=None): + exclude_fields = exclude_fields or [] # 如果未指定排除字段，默认为空列表 + + for value in values: + # 查找原表和测试表中该ID的相关数据 + rows_data = [row for row in list_doc_data if getattr(row, keys[0]) == value] + rows_test = [row for row in list_doc_data_test if getattr(row, keys[0]) == value] + + for row_data in rows_data: + # 在 doc_data_test 中查找相同主键的行 + row_test = next( + (row for row in rows_test if all(getattr(row, key) == getattr(row_data, key) for key in keys)), + None) + + if row_test: + # 如果在 doc_data_test 中找到相同的主键，则逐列比较 + columns = fields_to_compare if fields_to_compare else row_data._fields + columns = [col for col in columns if col not in exclude_fields] # 过滤排除字段 + + for column in columns: + value_data = getattr(row_data, column) + value_test = getattr(row_test, column) + if value_data != value_test: + differences.append({ + '主键': {key: getattr(row_data, key) for key in keys}, + '字段': column, + '生产表': f"\n{value_data}\n", + '测试表': f"\n{value_test}\n" + }) + + # 统计字段差异次数 + if column in field_diff_count: + field_diff_count[column] += 1 + else: + field_diff_count[column] = 1 + + else: + # 如果在 doc_data_test 中未找到相同的行 + differences.append({ + '主键': {key: getattr(row_data, key) for key in keys}, + '消息': f'在 {data_table_test} 中未找到该行' + }) + + # 比较 doc_data_test 中的行是否在 doc_data 中存在 + for row_test in rows_test: + row_data = next( + (row for row in rows_data if all(getattr(row, key) == getattr(row_test, key) for key in keys)), None) + if not row_data: + differences.append({ + '主键': {key: getattr(row_test, key) for key in keys}, + '消息': f'在 {data_table_pro} 中未找到该行' + }) + + +compare_data(fields_to_compare, exclude_fields) + +# 使用集合来保存唯一的 topicid +id_set = set() +field_set = set() + +grouped_id_dict = {} + +# 输出指定字段的差异 +with open(output_file, "a") as f: + if differences: + f.write("\n发现指定字段的差异:\n") + for diff in differences: + # 逐行打印每个差异 + f.write(f"主键: {diff['主键']}\n") + f.write(f"字段: {diff.get('字段', 'N/A')}\n") + f.write(f"生产表: {diff.get('生产表', 'N/A')}\n") + f.write(f"测试表: {diff.get('测试表', 'N/A')}\n") + f.write("-" * 50) # 分隔符，便于查看 + f.write("\n") + + # 将差异ID按字段名分组 + field = diff.get('字段', '未分组') + if field not in grouped_id_dict: + grouped_id_dict[field] = set() + for key in keys: + id = diff['主键'][key] + id_set.add('"' + id + '",') + grouped_id_dict[field].add('"' + id + '",') + + # 输出分组后的差异ID + if grouped_id_dict: + f.write("\n差异ID按字段分组如下:") + for field, ids in grouped_id_dict.items(): + field_set.add('"' + field + '",') + f.write(f"字段: {field}\n") + f.write("差异ID:") + for id in ids: + f.write(id) + f.write("\n") + f.write("-" * 50) # 分隔符，便于查看 + f.write("\n") + else: + f.write("\n指定字段未发现差异。\n") + + f.write("\n") + + # 只有在 field_set 不为空时才打印 + if field_set: + f.write("\n存在差异的 字段 为:\n") + # 打印所有唯一的 field + for field in field_set: + f.write(field + "\n") + + # 只有在 id_set 不为空时才打印 + if id_set: + f.write("\n存在差异的 ID 为:\n") + # 打印所有唯一的 topicid + for topicid in id_set: + f.write(topicid + "\n") + f.write("\n") + + # 计算存在差异的 ID + different_ids = {id.strip('"').strip(',').strip('"') for id in id_set} + + # 计算不存在差异的 ID（即在 values 中但不在 different_ids 中） + non_different_ids = set(values) - different_ids + + # 只有在 non_different_ids 非空时才打印 + if non_different_ids: + f.write("\n不存在差异的 ID 为:\n") + for topicid in non_different_ids: + f.write(f'"{topicid}",\n') + f.write("\n") + + f.write("总计key " + len(values).__str__() + "个") + # 统计每个字段的差异数量 + if field_diff_count: + f.write("\n字段差异统计如下:\n") + for field, count in field_diff_count.items(): + f.write(f"字段 '{field}' 发现 {count} 处差异\n") + +# 关闭连接 +cluster.shutdown() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..23696d6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +Flask==2.3.3 +cassandra-driver==3.29.1 \ No newline at end of file diff --git a/static/js/app.js b/static/js/app.js new file mode 100644 index 0000000..2ad5934 --- /dev/null +++ b/static/js/app.js @@ -0,0 +1,1408 @@ +// 全局变量 +let currentResults = null; +let currentIdenticalPage = 1; +let identicalPageSize = 10; +let filteredIdenticalResults = []; + +// 页面加载完成后初始化 +document.addEventListener('DOMContentLoaded', function() { + loadDefaultConfig(); + loadConfigGroups(); // 加载配置组列表 +}); + +// 加载配置组列表 +async function loadConfigGroups() { + try { + const response = await fetch('/api/config-groups'); + const result = await response.json(); + + if (result.success) { + const select = document.getElementById('configGroupSelect'); + select.innerHTML = ''; + + result.data.forEach(group => { + const option = document.createElement('option'); + option.value = group.id; + option.textContent = `${group.name} (${group.description || '无描述'})`; + select.appendChild(option); + }); + + if (result.data.length === 0) { + console.log('暂无配置组，数据库已就绪'); + } + } else { + console.error('加载配置组失败:', result.error); + // 如果是数据库问题，显示友好提示 + if (result.error && result.error.includes('table')) { + showAlert('info', '正在初始化数据库，请稍后再试'); + } + } + } catch (error) { + console.error('加载配置组失败:', error); + showAlert('warning', '配置组功能暂时不可用，但不影响基本查询功能'); + } +} + +// 加载选中的配置组 +async function loadSelectedConfigGroup() { + const groupId = document.getElementById('configGroupSelect').value; + + if (!groupId) { + showAlert('warning', '请先选择一个配置组'); + return; + } + + try { + const response = await fetch(`/api/config-groups/${groupId}`); + const result = await response.json(); + + if (result.success) { + const config = result.data; + + // 填充生产环境配置 + document.getElementById('pro_cluster_name').value = config.pro_config.cluster_name || ''; + document.getElementById('pro_datacenter').value = config.pro_config.datacenter || ''; + document.getElementById('pro_hosts').value = (config.pro_config.hosts || []).join(','); + document.getElementById('pro_port').value = config.pro_config.port || 9042; + document.getElementById('pro_username').value = config.pro_config.username || ''; + document.getElementById('pro_password').value = config.pro_config.password || ''; + document.getElementById('pro_keyspace').value = config.pro_config.keyspace || ''; + document.getElementById('pro_table').value = config.pro_config.table || ''; + + // 填充测试环境配置 + document.getElementById('test_cluster_name').value = config.test_config.cluster_name || ''; + document.getElementById('test_datacenter').value = config.test_config.datacenter || ''; + document.getElementById('test_hosts').value = (config.test_config.hosts || []).join(','); + document.getElementById('test_port').value = config.test_config.port || 9042; + document.getElementById('test_username').value = config.test_config.username || ''; + document.getElementById('test_password').value = config.test_config.password || ''; + document.getElementById('test_keyspace').value = config.test_config.keyspace || ''; + document.getElementById('test_table').value = config.test_config.table || ''; + + // 填充查询配置 + document.getElementById('keys').value = (config.query_config.keys || []).join(','); + document.getElementById('fields_to_compare').value = (config.query_config.fields_to_compare || []).join(','); + document.getElementById('exclude_fields').value = (config.query_config.exclude_fields || []).join(','); + + showAlert('success', `配置组 "${config.name}" 加载成功`); + } else { + showAlert('danger', result.error || '加载配置组失败'); + } + } catch (error) { + showAlert('danger', '加载配置组失败: ' + error.message); + } +} + +// 显示保存配置组对话框 +function showSaveConfigDialog() { + const modalContent = ` + + `; + + // 移除现有modal + const existingModal = document.getElementById('saveConfigModal'); + if (existingModal) { + existingModal.remove(); + } + + // 添加新modal到body + document.body.insertAdjacentHTML('beforeend', modalContent); + + // 显示modal + const modal = new bootstrap.Modal(document.getElementById('saveConfigModal')); + modal.show(); +} + +// 保存配置组 +async function saveConfigGroup() { + const name = document.getElementById('configGroupName').value.trim(); + const description = document.getElementById('configGroupDesc').value.trim(); + + if (!name) { + showAlert('warning', '请输入配置组名称'); + return; + } + + const config = getCurrentConfig(); + + try { + const response = await fetch('/api/config-groups', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + name: name, + description: description, + ...config + }) + }); + + const result = await response.json(); + + if (result.success) { + // 关闭modal + const modal = bootstrap.Modal.getInstance(document.getElementById('saveConfigModal')); + modal.hide(); + + // 重新加载配置组列表 + await loadConfigGroups(); + + showAlert('success', result.message); + } else { + showAlert('danger', result.error || '保存配置组失败'); + } + } catch (error) { + showAlert('danger', '保存配置组失败: ' + error.message); + } +} + +// 显示管理配置组对话框 +async function showManageConfigDialog() { + try { + const response = await fetch('/api/config-groups'); + const result = await response.json(); + + if (result.success) { + let configGroupsList = ''; + + if (result.data.length === 0) { + configGroupsList = '

暂无配置组

'; + } else { + result.data.forEach(group => { + const createdDate = new Date(group.created_at).toLocaleString(); + const updatedDate = new Date(group.updated_at).toLocaleString(); + + configGroupsList += ` +
+
+
+
+
${group.name}
+

${group.description || '无描述'}

+ 创建: ${createdDate} | 更新: ${updatedDate} +
+
+ + +
+
+
+
+ `; + }); + } + + const modalContent = ` + + `; + + // 移除现有modal + const existingModal = document.getElementById('manageConfigModal'); + if (existingModal) { + existingModal.remove(); + } + + // 添加新modal到body + document.body.insertAdjacentHTML('beforeend', modalContent); + + // 显示modal + const modal = new bootstrap.Modal(document.getElementById('manageConfigModal')); + modal.show(); + + } else { + showAlert('danger', '加载配置组列表失败'); + } + } catch (error) { + showAlert('danger', '加载配置组列表失败: ' + error.message); + } +} + +// 通过ID加载配置组 +async function loadConfigGroupById(groupId) { + try { + const response = await fetch(`/api/config-groups/${groupId}`); + const result = await response.json(); + + if (result.success) { + const config = result.data; + + // 填充表单数据（与loadSelectedConfigGroup相同逻辑） + document.getElementById('pro_cluster_name').value = config.pro_config.cluster_name || ''; + document.getElementById('pro_datacenter').value = config.pro_config.datacenter || ''; + document.getElementById('pro_hosts').value = (config.pro_config.hosts || []).join(','); + document.getElementById('pro_port').value = config.pro_config.port || 9042; + document.getElementById('pro_username').value = config.pro_config.username || ''; + document.getElementById('pro_password').value = config.pro_config.password || ''; + document.getElementById('pro_keyspace').value = config.pro_config.keyspace || ''; + document.getElementById('pro_table').value = config.pro_config.table || ''; + + document.getElementById('test_cluster_name').value = config.test_config.cluster_name || ''; + document.getElementById('test_datacenter').value = config.test_config.datacenter || ''; + document.getElementById('test_hosts').value = (config.test_config.hosts || []).join(','); + document.getElementById('test_port').value = config.test_config.port || 9042; + document.getElementById('test_username').value = config.test_config.username || ''; + document.getElementById('test_password').value = config.test_config.password || ''; + document.getElementById('test_keyspace').value = config.test_config.keyspace || ''; + document.getElementById('test_table').value = config.test_config.table || ''; + + document.getElementById('keys').value = (config.query_config.keys || []).join(','); + document.getElementById('fields_to_compare').value = (config.query_config.fields_to_compare || []).join(','); + document.getElementById('exclude_fields').value = (config.query_config.exclude_fields || []).join(','); + + // 更新下拉框选中状态 + document.getElementById('configGroupSelect').value = groupId; + + // 关闭管理modal + const modal = bootstrap.Modal.getInstance(document.getElementById('manageConfigModal')); + modal.hide(); + + showAlert('success', `配置组 "${config.name}" 加载成功`); + } else { + showAlert('danger', result.error || '加载配置组失败'); + } + } catch (error) { + showAlert('danger', '加载配置组失败: ' + error.message); + } +} + +// 删除配置组 +async function deleteConfigGroup(groupId, groupName) { + if (!confirm(`确定要删除配置组 "${groupName}" 吗？此操作不可撤销。`)) { + return; + } + + try { + const response = await fetch(`/api/config-groups/${groupId}`, { + method: 'DELETE' + }); + + const result = await response.json(); + + if (result.success) { + showAlert('success', result.message); + + // 重新加载配置组列表 + await loadConfigGroups(); + + // 重新显示管理对话框 + setTimeout(() => { + showManageConfigDialog(); + }, 500); + } else { + showAlert('danger', result.error || '删除配置组失败'); + } + } catch (error) { + showAlert('danger', '删除配置组失败: ' + error.message); + } +} + +// 加载默认配置 +async function loadDefaultConfig() { + try { + const response = await fetch('/api/default-config'); + const config = await response.json(); + + // 填充生产环境配置 + document.getElementById('pro_cluster_name').value = config.pro_config.cluster_name || ''; + document.getElementById('pro_datacenter').value = config.pro_config.datacenter || ''; + document.getElementById('pro_hosts').value = config.pro_config.hosts.join(','); + document.getElementById('pro_port').value = config.pro_config.port; + document.getElementById('pro_username').value = config.pro_config.username; + document.getElementById('pro_password').value = config.pro_config.password; + document.getElementById('pro_keyspace').value = config.pro_config.keyspace; + document.getElementById('pro_table').value = config.pro_config.table; + + // 填充测试环境配置 + document.getElementById('test_cluster_name').value = config.test_config.cluster_name || ''; + document.getElementById('test_datacenter').value = config.test_config.datacenter || ''; + document.getElementById('test_hosts').value = config.test_config.hosts.join(','); + document.getElementById('test_port').value = config.test_config.port; + document.getElementById('test_username').value = config.test_config.username; + document.getElementById('test_password').value = config.test_config.password; + document.getElementById('test_keyspace').value = config.test_config.keyspace; + document.getElementById('test_table').value = config.test_config.table; + + // 填充查询配置 + document.getElementById('keys').value = config.keys.join(','); + document.getElementById('fields_to_compare').value = config.fields_to_compare.join(','); + document.getElementById('exclude_fields').value = config.exclude_fields.join(','); + + showAlert('success', '默认配置加载成功'); + } catch (error) { + showAlert('danger', '加载默认配置失败: ' + error.message); + } +} + +// 获取当前配置 +function getCurrentConfig() { + return { + pro_config: { + cluster_name: document.getElementById('pro_cluster_name').value, + datacenter: document.getElementById('pro_datacenter').value, + hosts: document.getElementById('pro_hosts').value.split(',').map(h => h.trim()).filter(h => h), + port: parseInt(document.getElementById('pro_port').value) || 9042, + username: document.getElementById('pro_username').value, + password: document.getElementById('pro_password').value, + keyspace: document.getElementById('pro_keyspace').value, + table: document.getElementById('pro_table').value + }, + test_config: { + cluster_name: document.getElementById('test_cluster_name').value, + datacenter: document.getElementById('test_datacenter').value, + hosts: document.getElementById('test_hosts').value.split(',').map(h => h.trim()).filter(h => h), + port: parseInt(document.getElementById('test_port').value) || 9042, + username: document.getElementById('test_username').value, + password: document.getElementById('test_password').value, + keyspace: document.getElementById('test_keyspace').value, + table: document.getElementById('test_table').value + }, + keys: document.getElementById('keys').value.split(',').map(k => k.trim()).filter(k => k), + fields_to_compare: document.getElementById('fields_to_compare').value + .split(',').map(f => f.trim()).filter(f => f), + exclude_fields: document.getElementById('exclude_fields').value + .split(',').map(f => f.trim()).filter(f => f), + values: document.getElementById('query_values').value + .split('\n').map(v => v.trim()).filter(v => v) + }; +} + +// 执行查询比对 +async function executeQuery() { + const config = getCurrentConfig(); + + // 验证配置 + if (!config.values.length) { + showAlert('warning', '请输入查询Key值'); + return; + } + + if (!config.keys.length) { + showAlert('warning', '请输入主键字段'); + return; + } + + // 显示加载动画 + document.getElementById('loading').style.display = 'block'; + document.getElementById('results').style.display = 'none'; + + try { + const response = await fetch('/api/query', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(config) + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.error || '查询失败'); + } + + const results = await response.json(); + currentResults = results; + displayResults(results); + + } catch (error) { + showAlert('danger', '查询失败: ' + error.message); + } finally { + document.getElementById('loading').style.display = 'none'; + } +} + +// 显示查询结果 +function displayResults(results) { + // 显示统计信息 + displayStats(results); + + // 更新选项卡计数 + document.getElementById('diff-count').textContent = results.differences.length; + document.getElementById('identical-count').textContent = results.identical_results.length; + + // 初始化相同结果分页数据 + filteredIdenticalResults = results.identical_results; + currentIdenticalPage = 1; + + // 显示各个面板内容 + displayDifferences(results.differences); + displayIdenticalResults(); + displayComparisonSummary(results.summary); + + // 显示结果区域 + document.getElementById('results').style.display = 'block'; + + showAlert('success', `查询完成！共处理${results.total_keys}个Key，发现${results.differences.length}处差异，${results.identical_results.length}条记录完全相同`); +} + +// 显示统计信息 +function displayStats(results) { + const statsHtml = ` +
+
+

${results.total_keys}

+

总Key数量

+
+
+
+
+

${results.identical_results.length}

+

相同记录

+
+
+
+
+

${results.differences.length}

+

差异记录

+
+
+
+
+

${Math.round((results.identical_results.length / results.total_keys) * 100)}%

+

一致性比例

+
+
+ `; + + document.getElementById('stats').innerHTML = statsHtml; +} + +// 显示差异详情 +function displayDifferences(differences) { + const differencesContainer = document.getElementById('differences'); + + if (!differences.length) { + differencesContainer.innerHTML = '

未发现差异

'; + return; + } + + let html = ''; + differences.forEach((diff, index) => { + if (diff.message) { + // 记录不存在的情况 + html += ` +
+ 差异 #${index + 1} +

主键: ${JSON.stringify(diff.key)}

+

${diff.message}

+
+ `; + } else { + // 字段值差异的情况 + const isJson = diff.is_json; + const isArray = diff.is_array; + const jsonClass = isJson ? 'json-field' : ''; + + html += ` +
+
+
+ 差异 #${index + 1} + ${isJson ? 'JSON字段' : ''} + ${isArray ? '数组字段' : ''} +
+ +
+

主键: ${JSON.stringify(diff.key)}

+

字段: ${diff.field}

+ +
+
+ + ${diff.field} +
+ + +
+
+
+ 生产环境 +
+
+
+
+ 
${escapeHtml(diff.pro_value)}
+ +
+
+
+ + +
+
+
+ 测试环境 +
+
+
+
+ 
${escapeHtml(diff.test_value)}
+ +
+
+
+
+
+ `; + } + }); + + differencesContainer.innerHTML = html; +} + +// HTML转义函数，防止XSS +function escapeHtml(text) { + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; +} + +// 显示相同结果 +function displayIdenticalResults() { + const identicalContainer = document.getElementById('identical-results'); + + if (!filteredIdenticalResults.length) { + identicalContainer.innerHTML = '

没有完全相同的记录

'; + return; + } + + // 计算分页 + const totalPages = Math.ceil(filteredIdenticalResults.length / identicalPageSize); + const startIndex = (currentIdenticalPage - 1) * identicalPageSize; + const endIndex = startIndex + identicalPageSize; + const currentPageData = filteredIdenticalResults.slice(startIndex, endIndex); + + let html = ` + +
+
+
+ + + 共 ${filteredIdenticalResults.length} 条记录 +
+
+
+
+ ${generatePagination(currentIdenticalPage, totalPages)} +
+
+
+ + +
+
+ + +
+
+ `; + + // 显示当前页数据 + currentPageData.forEach((result, index) => { + const globalIndex = startIndex + index + 1; + html += ` +
+
+
+
+ 相同记录 #${globalIndex} + 完全匹配 +
+
+ + +
+
+

主键: ${JSON.stringify(result.key)}

+
+
+
+ `; + + // 显示字段对比，左右布局 + const proFields = result.pro_fields || {}; + const testFields = result.test_fields || {}; + const allFields = new Set([...Object.keys(proFields), ...Object.keys(testFields)]); + + html += '
字段对比 (生产环境 vs 测试环境):
'; + + allFields.forEach(fieldName => { + const proValue = proFields[fieldName] || ''; + const testValue = testFields[fieldName] || ''; + const isJson = (proValue.includes('{') || proValue.includes('[')) || + (testValue.includes('{') || testValue.includes('[')); + + html += ` +
+
+ + ${fieldName} + ${isJson ? 'JSON' : ''} +
+ + +
+
+
+ 生产环境 +
+
+
+
+ 
${escapeHtml(proValue)}
+ +
+
+
+ + +
+
+
+ 测试环境 +
+
+
+
+ 
${escapeHtml(testValue)}
+ +
+
+
+
+ `; + }); + + html += ` +
+
+
+ `; + }); + + // 底部分页 + if (totalPages > 1) { + html += ` +
+
+
+ ${generatePagination(currentIdenticalPage, totalPages)} +
+
+
+ `; + } + + identicalContainer.innerHTML = html; +} + +// 生成分页导航 +function generatePagination(currentPage, totalPages) { + if (totalPages <= 1) return ''; + + let html = ''; + return html; +} + +// 跳转到指定页面 +function goToIdenticalPage(page) { + const totalPages = Math.ceil(filteredIdenticalResults.length / identicalPageSize); + if (page < 1 || page > totalPages) return; + + currentIdenticalPage = page; + displayIdenticalResults(); +} + +// 改变每页显示数量 +function changePageSize(newSize) { + identicalPageSize = parseInt(newSize); + currentIdenticalPage = 1; + displayIdenticalResults(); +} + +// 搜索相同结果 +function searchIdenticalResults(searchTerm) { + if (!currentResults) return; + + if (!searchTerm.trim()) { + filteredIdenticalResults = currentResults.identical_results; + } else { + const term = searchTerm.toLowerCase(); + filteredIdenticalResults = currentResults.identical_results.filter(result => { + // 搜索主键 + const keyStr = JSON.stringify(result.key).toLowerCase(); + if (keyStr.includes(term)) return true; + + // 搜索字段内容 + const proFields = result.pro_fields || {}; + const testFields = result.test_fields || {}; + const allValues = [...Object.values(proFields), ...Object.values(testFields)]; + + return allValues.some(value => + String(value).toLowerCase().includes(term) + ); + }); + } + + currentIdenticalPage = 1; + displayIdenticalResults(); +} + +// 复制到剪贴板 +function copyToClipboard(text, button) { + navigator.clipboard.writeText(text).then(function() { + // 显示复制成功反馈 + const originalIcon = button.innerHTML; + button.innerHTML = ''; + button.classList.add('btn-success'); + button.classList.remove('btn-outline-secondary'); + + setTimeout(() => { + button.innerHTML = originalIcon; + button.classList.remove('btn-success'); + button.classList.add('btn-outline-secondary'); + }, 1000); + }).catch(function(err) { + console.error('复制失败:', err); + showAlert('danger', '复制失败，请手动复制'); + }); +} + +// JS字符串转义 +function escapeForJs(str) { + return str.replace(/'/g, "\\'").replace(/"/g, '\\"').replace(/\n/g, '\\n').replace(/\r/g, '\\r'); +} + +// 显示原生数据 +function showRawData(keyStr) { + if (!currentResults) return; + + try { + const key = JSON.parse(keyStr); + const keyValue = Object.values(key)[0]; + + // 在原生数据中查找匹配的记录 + const proRecord = currentResults.raw_pro_data.find(record => + Object.values(record).includes(keyValue) + ); + const testRecord = currentResults.raw_test_data.find(record => + Object.values(record).includes(keyValue) + ); + + let modalContent = ` + + `; + + // 移除现有modal + const existingModal = document.getElementById('rawDataModal'); + if (existingModal) { + existingModal.remove(); + } + + // 添加新modal到body + document.body.insertAdjacentHTML('beforeend', modalContent); + + // 显示modal + const modal = new bootstrap.Modal(document.getElementById('rawDataModal')); + modal.show(); + + } catch (e) { + showAlert('danger', '无法显示原生数据：' + e.message); + } +} + +// 显示差异数据的原生数据 +function showDifferenceRawData(keyStr) { + showRawData(keyStr); // 复用相同的原生数据显示逻辑 +} + +// 显示比较总结 +function displayComparisonSummary(summary) { + const summaryContainer = document.getElementById('comparison-summary'); + + if (!summary) { + summaryContainer.innerHTML = '

无总结数据

'; + return; + } + + const qualityLevel = summary.data_quality.quality_level; + + const html = ` + +
+
+
+
+
数据概览
+
+
+
+
+

${summary.overview.total_keys_queried}

+ 查询Key总数 +
+
+

${summary.overview.identical_records}

+ 相同记录 +
+
+

${summary.overview.different_records}

+ 差异记录 +
+
+
+
+
+
+
+
+
数据质量评估
+
+
+

${qualityLevel.level}

+

${qualityLevel.description}

+
+
+ ${summary.data_quality.consistency_score}% +
+
+ 数据一致性评分 +
+
+
+
+ + +
+
+
+
+
百分比统计
+
+
+
+
+ 数据一致性: + ${summary.percentages.data_consistency}% +
+
+
+
+
+
+
+ 数据差异率: + ${summary.percentages.data_differences}% +
+
+
+
+
+
+
+ 数据缺失率: + ${summary.percentages.missing_rate}% +
+
+
+
+
+
+
+
+
+
+
+
字段差异TOP5
+
+
+ ${summary.field_analysis.most_different_fields.length > 0 ? + summary.field_analysis.most_different_fields.map(([field, count]) => ` +
+ ${field} + ${count}次 +
+ `).join('') : + '

无字段差异统计

' + } +
+
+
+
+ + +
+
+
改进建议
+
+
+
    + ${summary.recommendations.map(recommendation => ` +
  • + + ${recommendation} +
    • + `).join('')} +
+
+
+ `; + + summaryContainer.innerHTML = html; +} + +// 显示字段差异统计 +function displayFieldStats(fieldStats) { + const fieldStatsContainer = document.getElementById('field_stats'); + + if (!Object.keys(fieldStats).length) { + fieldStatsContainer.innerHTML = '

无字段差异统计

'; + return; + } + + let html = '
'; + html += ''; + + const totalDiffs = Object.values(fieldStats).reduce((sum, count) => sum + count, 0); + + Object.entries(fieldStats) + .sort(([, a], [, b]) => b - a) + .forEach(([field, count]) => { + const percentage = ((count / totalDiffs) * 100).toFixed(1); + html += ` + + + + + + `; + }); + + html += '
字段名差异次数占比
${field}${count}${percentage}%
'; + fieldStatsContainer.innerHTML = html; +} + +// 清空结果 +function clearResults() { + document.getElementById('results').style.display = 'none'; + document.getElementById('query_values').value = ''; + currentResults = null; + showAlert('info', '结果已清空'); +} + +// 导出配置 +function exportConfig() { + const config = getCurrentConfig(); + const dataStr = JSON.stringify(config, null, 2); + const dataBlob = new Blob([dataStr], {type: 'application/json'}); + + const link = document.createElement('a'); + link.href = URL.createObjectURL(dataBlob); + link.download = 'db_compare_config.json'; + link.click(); +} + +// 导出结果 +function exportResults() { + if (!currentResults) { + showAlert('warning', '没有可导出的结果'); + return; + } + + let output = `数据库查询比对结果\n`; + output += `生成时间: ${new Date().toLocaleString()}\n`; + output += `=`.repeat(50) + '\n\n'; + + output += `统计信息:\n`; + output += `- 总Key数量: ${currentResults.total_keys}\n`; + output += `- 生产表记录: ${currentResults.pro_count}\n`; + output += `- 测试表记录: ${currentResults.test_count}\n`; + output += `- 发现差异: ${currentResults.differences.length}\n\n`; + + if (currentResults.differences.length > 0) { + output += `差异详情:\n`; + output += `-`.repeat(30) + '\n'; + + currentResults.differences.forEach((diff, index) => { + output += `差异 #${index + 1}:\n`; + output += `主键: ${JSON.stringify(diff.key)}\n`; + + if (diff.message) { + output += `消息: ${diff.message}\n`; + } else { + output += `字段: ${diff.field}\n`; + output += `生产表值: ${diff.pro_value}\n`; + output += `测试表值: ${diff.test_value}\n`; + } + output += '\n'; + }); + + // 字段差异统计 + if (Object.keys(currentResults.field_diff_count).length > 0) { + output += `字段差异统计:\n`; + output += `-`.repeat(30) + '\n'; + Object.entries(currentResults.field_diff_count) + .sort(([, a], [, b]) => b - a) + .forEach(([field, count]) => { + output += `${field}: ${count}次\n`; + }); + } + } + + const dataBlob = new Blob([output], {type: 'text/plain;charset=utf-8'}); + const link = document.createElement('a'); + link.href = URL.createObjectURL(dataBlob); + link.download = `db_compare_result_${new Date().toISOString().slice(0, 19).replace(/:/g, '-')}.txt`; + link.click(); +} + +// 显示一键导入对话框 +function showImportDialog(env) { + const envName = env === 'pro' ? '生产环境' : '测试环境'; + + const modalContent = ` + + `; + + // 移除现有modal + const existingModal = document.getElementById('importModal'); + if (existingModal) { + existingModal.remove(); + } + + // 添加新modal到body + document.body.insertAdjacentHTML('beforeend', modalContent); + + // 显示modal + const modal = new bootstrap.Modal(document.getElementById('importModal')); + modal.show(); +} + +// 导入配置数据 +function importConfig(env) { + const configData = document.getElementById('importConfigData').value.trim(); + + if (!configData) { + showAlert('warning', '请输入配置数据'); + return; + } + + try { + // 解析配置数据 + const config = parseConfigData(configData); + + // 根据环境类型填充对应的表单字段 + const prefix = env === 'pro' ? 'pro' : 'test'; + + if (config.clusterName) { + document.getElementById(`${prefix}_cluster_name`).value = config.clusterName; + } + if (config.datacenter) { + document.getElementById(`${prefix}_datacenter`).value = config.datacenter; + } + if (config.clusterNodes) { + document.getElementById(`${prefix}_hosts`).value = config.clusterNodes; + } + if (config.port) { + document.getElementById(`${prefix}_port`).value = config.port; + } + if (config.username) { + document.getElementById(`${prefix}_username`).value = config.username; + } + if (config.password) { + document.getElementById(`${prefix}_password`).value = config.password; + } + if (config.keyspace) { + document.getElementById(`${prefix}_keyspace`).value = config.keyspace; + } + if (config.table) { + document.getElementById(`${prefix}_table`).value = config.table; + } + + // 关闭modal + const modal = bootstrap.Modal.getInstance(document.getElementById('importModal')); + modal.hide(); + + const envName = env === 'pro' ? '生产环境' : '测试环境'; + showAlert('success', `${envName}配置导入成功！`); + + } catch (error) { + showAlert('danger', '配置解析失败：' + error.message); + } +} + +// 解析配置数据 +function parseConfigData(configText) { + const config = {}; + + try { + // 尝试按JSON格式解析 + const jsonConfig = JSON.parse(configText); + return jsonConfig; + } catch (e) { + // 如果不是JSON，按YAML格式解析 + const lines = configText.split('\n'); + + for (const line of lines) { + const trimmedLine = line.trim(); + if (!trimmedLine || trimmedLine.startsWith('#')) { + continue; // 跳过空行和注释 + } + + const colonIndex = trimmedLine.indexOf(':'); + if (colonIndex === -1) { + continue; // 跳过无效行 + } + + const key = trimmedLine.substring(0, colonIndex).trim(); + let value = trimmedLine.substring(colonIndex + 1).trim(); + + // 处理带引号的值 + if ((value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'"))) { + value = value.slice(1, -1); + } + + // 尝试转换数字 + if (!isNaN(value) && value !== '') { + config[key] = parseInt(value); + } else { + config[key] = value; + } + } + + return config; + } +} + +// 显示提示信息 +function showAlert(type, message) { + // 移除已存在的提示 + const existingAlert = document.querySelector('.alert'); + if (existingAlert) { + existingAlert.remove(); + } + + const alertHtml = ` + + `; + + // 插入到页面顶部 + const container = document.querySelector('.container-fluid'); + container.insertAdjacentHTML('afterbegin', alertHtml); + + // 自动隐藏 + setTimeout(() => { + const alert = document.querySelector('.alert'); + if (alert) { + alert.remove(); + } + }, 5000); +} \ No newline at end of file diff --git a/templates/db_compare.html b/templates/db_compare.html new file mode 100644 index 0000000..1f46dca --- /dev/null +++ b/templates/db_compare.html @@ -0,0 +1,399 @@ + + + + + + 数据库查询比对工具 + + + + + + + + +
+
+
+

+ 数据库查询比对工具 +

+
+
+ +
+ +
+
+

配置管理

+ + +
+
+
配置组管理
+
+
+
+
+ +
+
+ +
+
+
+
+ +
+
+ +
+
+
+
+ +
+ + +
+ + +
+
+
生产环境配置
+ +
+
+
+
+ + +
+
+ + +
+
+
+
+ + +
+
+ + +
+
+
+
+ + +
+
+ + +
+
+
+
+ + +
+
+ + +
+
+
+
+ + +
+
+
测试环境配置
+ +
+
+
+
+ + +
+
+ + +
+
+
+
+ + +
+
+ + +
+
+
+
+ + +
+
+ + +
+
+
+
+ + +
+
+ + +
+
+
+
+ + +
+
+
查询配置
+
+
+
+ + +
+
+ + +
+
+ + +
+
+
+
+
+ + +
+
+

查询Key管理

+
+ + +
+
+ + +
+ + +
+ +

正在执行查询比对...

+
+
+ + + +
+
+
+ + + + + \ No newline at end of file diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..270a7f3 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,374 @@ + + + + + + 大数据工具集合 + + + + + + + + + +
+
+

+ 大数据工具集合 +

+

+ 专业的数据处理、分析和比对工具平台
+ 提升数据工作效率，简化复杂操作 +

+
+
+ + +
+
+
+
+
+ 1 + 可用工具 +
+
+
+
+ 100% + 可视化操作 +
+
+
+
+ 0 + 学习成本 +
+
+
+
+
+ + +
+
+
+
+
+

可用工具

+

选择合适的工具来处理您的数据任务

+
+
+
+ +
+ +
+
+
+
可用
+
+ +
+

数据库查询比对工具

+

+ 专业的Cassandra数据库比对工具，支持生产环境与测试环境数据差异分析， + 提供批量查询、字段级比对和详细统计报告。 +

+
+ +
+
核心功能：
+
    +
  • 支持多环境数据库配置管理
    • +
  • 批量Key查询和数据比对
    • +
  • 自定义比较字段和排除字段
    • +
  • 可视化差异展示和统计
    • +
  • 配置和结果导出功能
    • +
+
+ +
+ + 立即使用 + +
+
+
+ + +
+
+
+
即将推出
+
+ +
+

数据分析工具

+

+ 强大的数据分析和可视化工具，支持多种数据源， + 提供丰富的图表类型和统计分析功能。 +

+
+ +
+
计划功能：
+
    +
  • 多数据源连接支持
    • +
  • 交互式图表生成
    • +
  • 自定义报表制作
    • +
  • 数据趋势分析
    • +
  • 自动化报告生成
    • +
+
+ +
+ +
+
+
+
+ + +
+
+
+
+
即将推出
+
+ +
+

数据导入导出工具

+

+ 高效的数据迁移工具，支持多种格式和数据库类型之间的数据传输， + 提供批量处理和进度监控功能。 +

+
+ +
+
计划功能：
+
    +
  • 多格式数据支持
    • +
  • 批量数据处理
    • +
  • 实时进度监控
    • +
  • 数据映射配置
    • +
  • 错误处理和日志
    • +
+
+ +
+ +
+
+
+ +
+
+
+
即将推出
+
+ +
+

数据质量检测工具

+

+ 专业的数据质量评估工具，自动检测数据完整性、一致性和准确性问题， + 生成详细的质量报告和改进建议。 +

+
+ +
+
计划功能：
+
    +
  • 数据完整性检查
    • +
  • 重复数据检测
    • +
  • 数据格式验证
    • +
  • 质量评分系统
    • +
  • 自动化修复建议
    • +
+
+ +
+ +
+
+
+
+
+
+ + +
+
+

© 2024 大数据工具集合. 专注于提供高效的数据处理解决方案.

+
+
+ + + + \ No newline at end of file