From d42cefd9ca44a04ba365be251990c195a08adcd0 Mon Sep 17 00:00:00 2001 From: YoVinchen Date: Mon, 11 Aug 2025 09:34:29 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E5=96=84=E9=A1=B5=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app.py | 13 +- modules/api_routes.py | 26 ++++ static/js/app.js | 353 ++++++++++++++++++++++++++++++++---------- templates/index.html | 79 ---------- 4 files changed, 306 insertions(+), 165 deletions(-) diff --git a/app.py b/app.py index eeb5d99..da5e115 100644 --- a/app.py +++ b/app.py @@ -76,6 +76,15 @@ if __name__ == '__main__': logger.info("=== BigDataTool 启动 ===") logger.info("应用架构：模块化") logger.info("支持功能：单表查询、分表查询、多主键查询、配置管理、查询历史") - logger.info("访问地址：http://localhost:5000") + + # 从环境变量获取配置，支持Docker部署 + import os + host = os.getenv('FLASK_HOST', '0.0.0.0') + port = int(os.getenv('FLASK_PORT', 5000)) + debug = os.getenv('FLASK_DEBUG', 'True').lower() == 'true' + + logger.info(f"访问地址：http://{host}:{port}") logger.info("API文档：/api/* 路径下的所有端点") - app.run(debug=True, port=5000) \ No newline at end of file + logger.info(f"配置信息 - 主机: {host}, 端口: {port}, 调试: {debug}") + + app.run(debug=debug, host=host, port=port) \ No newline at end of file diff --git a/modules/api_routes.py b/modules/api_routes.py index 67d66f3..acd6929 100644 --- a/modules/api_routes.py +++ b/modules/api_routes.py @@ -68,6 +68,32 @@ def setup_routes(app, query_log_collector): return render_template('redis_test.html') # 基础API + @app.route('/api/health') + def health_check(): + """健康检查端点，用于Docker健康检查和服务监控""" + try: + # 检查应用基本状态 + current_time = datetime.now().isoformat() + + # 简单的数据库连接检查（可选） + from .database import ensure_database + db_status = ensure_database() + + return jsonify({ + 'status': 'healthy', + 'timestamp': current_time, + 'service': 'BigDataTool', + 'version': '2.0', + 'database': 'ok' if db_status else 'warning' + }) + except Exception as e: + logger.error(f"健康检查失败: {str(e)}") + return jsonify({ + 'status': 'unhealthy', + 'error': str(e), + 'timestamp': datetime.now().isoformat() + }), 503 + @app.route('/api/default-config') def get_default_config(): return jsonify(DEFAULT_CONFIG) diff --git a/static/js/app.js b/static/js/app.js index b6bd977..46ab4d0 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -908,35 +908,65 @@ function displayStats(results) { function displayDifferences() { const differencesContainer = document.getElementById('differences'); + // 保存当前搜索框的值 + const currentSearchValue = document.getElementById('differenceSearch')?.value || ''; + if (!filteredDifferenceResults.length) { differencesContainer.innerHTML = '

未发现差异

'; return; } - // 计算分页 - const totalPages = Math.ceil(filteredDifferenceResults.length / differencePageSize); + // 按主键分组差异 + const groupedDifferences = groupDifferencesByKey(filteredDifferenceResults); + const totalGroups = Object.keys(groupedDifferences).length; + + // 计算分页（基于主键组） + const groupKeys = Object.keys(groupedDifferences); + const totalPages = Math.ceil(groupKeys.length / differencePageSize); const startIndex = (currentDifferencePage - 1) * differencePageSize; const endIndex = startIndex + differencePageSize; - const currentPageData = filteredDifferenceResults.slice(startIndex, endIndex); + const currentPageKeys = groupKeys.slice(startIndex, endIndex); + + // 统计字段差异类型 + const fieldStats = {}; + filteredDifferenceResults.forEach(diff => { + if (diff.field) { + fieldStats[diff.field] = (fieldStats[diff.field] || 0) + 1; + } + }); let html = ` + +
+
+ + ${Object.entries(fieldStats).map(([field, count]) => ` + + `).join('')} +
+
+
- 共 ${filteredDifferenceResults.length} 条差异记录 + 共 ${totalGroups} 个主键组，${filteredDifferenceResults.length} 条差异记录
@@ -951,94 +981,111 @@ function displayDifferences() {
+ onkeypress="if(event.key === 'Enter') searchDifferenceResults(this.value)" + id="differenceSearch" + value="${currentSearchValue}"> + +
`; - // 显示当前页数据 - currentPageData.forEach((diff, index) => { - const globalIndex = startIndex + index + 1; - if (diff.message) { - // 记录不存在的情况 - html += ` -
-
-
-
- 差异 #${globalIndex} - 记录缺失 -
-
- -
+ // 显示当前页的主键组 + currentPageKeys.forEach((key, groupIndex) => { + const diffs = groupedDifferences[key]; + const globalIndex = startIndex + groupIndex + 1; + + html += ` +
+
+
+
+ 主键组 #${globalIndex} + ${diffs.length} 个差异字段 +
+
+ +
-

主键: ${formatCompositeKey(diff.key)}

-
-
-

${diff.message}

+

主键: ${formatCompositeKey(key)}

- `; - } else { - // 字段值差异的情况 - const isJson = diff.is_json; - const isArray = diff.is_array; - const jsonClass = isJson ? 'json-field' : ''; - - html += ` -
-
-
-
- 差异 #${globalIndex} - 字段差异 - ${isJson ? 'JSON字段' : ''} - ${isArray ? '数组字段' : ''} -
-
- - -
-
-

主键: ${formatCompositeKey(diff.key)}

-

差异字段: ${diff.field}

+
+
+ `; + + // 显示该主键下的所有差异字段 + diffs.forEach((diff, diffIndex) => { + if (diff.message) { + // 记录不存在的情况 + html += ` +
+ ${diff.message}
-
-
-
-
- - ${diff.field} + `; + } else { + // 字段值差异的情况 + const isJson = diff.is_json; + const isArray = diff.is_array; + const jsonClass = isJson ? 'json-field' : ''; + const fieldId = `field_${globalIndex}_${diffIndex}`; + + html += ` +
+
+
+ ${diff.field} + ${isJson ? 'JSON' : ''} + ${isArray ? '数组' : ''} +
+ +
+
+
+
+
+
+ 生产环境 +
+
+ 
${escapeHtml(diff.pro_value)}
+
+
- - -
-
-
- 
${escapeHtml(diff.pro_value)}
-${escapeHtml(diff.test_value)}
- +
+
+
+ 测试环境 +
+
+ 
${escapeHtml(diff.test_value)}
+ `; + } + }); + + html += ` +
- `; - } +
+ `; }); // 底部分页 @@ -1055,6 +1102,15 @@ ${escapeHtml(diff.test_value)} } differencesContainer.innerHTML = html; + + // 恢复搜索框的焦点和光标位置 + if (currentSearchValue) { + const searchInput = document.getElementById('differenceSearch'); + if (searchInput) { + searchInput.focus(); + searchInput.setSelectionRange(searchInput.value.length, searchInput.value.length); + } + } } // HTML转义函数，防止XSS @@ -1068,6 +1124,9 @@ function escapeHtml(text) { function displayIdenticalResults() { const identicalContainer = document.getElementById('identical-results'); + // 保存当前搜索框的值 + const currentSearchValue = document.getElementById('identicalSearch')?.value || ''; + if (!filteredIdenticalResults.length) { identicalContainer.innerHTML = '

没有完全相同的记录

'; return; @@ -1111,7 +1170,15 @@ function displayIdenticalResults() {
+ onkeypress="if(event.key === 'Enter') searchIdenticalResults(this.value)" + id="identicalSearch" + value="${currentIdenticalSearchTerm || ''}"> + +
`; @@ -1219,6 +1286,15 @@ ${escapeHtml(String(testValue))} } identicalContainer.innerHTML = html; + + // 恢复搜索框的焦点和光标位置 + if (currentSearchValue) { + const searchInput = document.getElementById('identicalSearch'); + if (searchInput) { + searchInput.focus(); + searchInput.setSelectionRange(searchInput.value.length, searchInput.value.length); + } + } } // 生成分页导航 @@ -1334,9 +1410,14 @@ function handleCustomPageSizeEnter(event) { } // 搜索相同结果 +let currentIdenticalSearchTerm = ''; + function searchIdenticalResults(searchTerm) { if (!currentResults) return; + // 保存搜索词 + currentIdenticalSearchTerm = searchTerm; + if (!searchTerm.trim()) { filteredIdenticalResults = currentResults.identical_results; } else { @@ -1361,6 +1442,15 @@ function searchIdenticalResults(searchTerm) { displayIdenticalResults(); } +// 清除相同记录搜索 +function clearIdenticalSearch() { + currentIdenticalSearchTerm = ''; + document.getElementById('identicalSearch').value = ''; + filteredIdenticalResults = currentResults.identical_results; + currentIdenticalPage = 1; + displayIdenticalResults(); +} + // 生成差异分页导航 function generateDifferencePagination(currentPage, totalPages) { if (totalPages <= 1) return ''; @@ -1474,14 +1564,28 @@ function handleCustomDiffPageSizeEnter(event) { } // 搜索差异结果 +let currentSearchTerm = ''; + function searchDifferenceResults(searchTerm) { if (!currentResults) return; + // 保存搜索词 + currentSearchTerm = searchTerm; + if (!searchTerm.trim()) { - filteredDifferenceResults = currentResults.differences; + // 如果有字段筛选，应用字段筛选；否则显示全部 + if (currentFieldFilter) { + filteredDifferenceResults = currentResults.differences.filter(diff => diff.field === currentFieldFilter); + } else { + filteredDifferenceResults = currentResults.differences; + } } else { const term = searchTerm.toLowerCase(); - filteredDifferenceResults = currentResults.differences.filter(diff => { + let baseResults = currentFieldFilter ? + currentResults.differences.filter(diff => diff.field === currentFieldFilter) : + currentResults.differences; + + filteredDifferenceResults = baseResults.filter(diff => { // 搜索主键 const keyStr = JSON.stringify(diff.key).toLowerCase(); if (keyStr.includes(term)) return true; @@ -1504,6 +1608,22 @@ function searchDifferenceResults(searchTerm) { displayDifferences(); } +// 清除差异搜索 +function clearDifferenceSearch() { + currentSearchTerm = ''; + document.getElementById('differenceSearch').value = ''; + + // 重新应用字段筛选 + if (currentFieldFilter) { + filteredDifferenceResults = currentResults.differences.filter(diff => diff.field === currentFieldFilter); + } else { + filteredDifferenceResults = currentResults.differences; + } + + currentDifferencePage = 1; + displayDifferences(); +} + // 复制到剪贴板 function copyToClipboard(text, button) { navigator.clipboard.writeText(text).then(function() { @@ -4271,4 +4391,69 @@ async function batchDeleteQueryHistory() { console.error('批量删除Cassandra查询历史记录失败:', error); showAlert('danger', `批量删除失败: ${error.message}`); } +} + +// 按主键分组差异数据 +function groupDifferencesByKey(differences) { + const grouped = {}; + differences.forEach(diff => { + const keyStr = typeof diff.key === 'object' ? JSON.stringify(diff.key) : diff.key; + if (!grouped[keyStr]) { + grouped[keyStr] = []; + } + grouped[keyStr].push(diff); + }); + return grouped; +} + +// 按字段筛选差异 +let currentFieldFilter = ''; + +function filterByField(field) { + currentFieldFilter = field; + + // 更新按钮状态 + const buttons = document.querySelectorAll('.btn-group button'); + buttons.forEach(btn => { + btn.classList.remove('active', 'btn-outline-primary'); + btn.classList.add('btn-outline-secondary'); + }); + + // 设置当前选中的按钮 + if (field === '') { + buttons[0].classList.remove('btn-outline-secondary'); + buttons[0].classList.add('btn-outline-primary', 'active'); + } else { + buttons.forEach(btn => { + if (btn.textContent.includes(field + ' (')) { + btn.classList.remove('btn-outline-secondary'); + btn.classList.add('btn-outline-primary', 'active'); + } + }); + } + + // 筛选差异数据（考虑搜索条件） + let baseResults = field === '' ? + currentResults.differences : + currentResults.differences.filter(diff => diff.field === field); + + // 如果有搜索条件，应用搜索过滤 + if (currentSearchTerm && currentSearchTerm.trim()) { + const term = currentSearchTerm.toLowerCase(); + filteredDifferenceResults = baseResults.filter(diff => { + const keyStr = JSON.stringify(diff.key).toLowerCase(); + if (keyStr.includes(term)) return true; + if (diff.field && diff.field.toLowerCase().includes(term)) return true; + if (diff.pro_value && String(diff.pro_value).toLowerCase().includes(term)) return true; + if (diff.test_value && String(diff.test_value).toLowerCase().includes(term)) return true; + if (diff.message && diff.message.toLowerCase().includes(term)) return true; + return false; + }); + } else { + filteredDifferenceResults = baseResults; + } + + // 重置页码并重新显示 + currentDifferencePage = 1; + displayDifferences(); } \ No newline at end of file diff --git a/templates/index.html b/templates/index.html index 3bc2ce0..d4dbe15 100644 --- a/templates/index.html +++ b/templates/index.html @@ -295,85 +295,6 @@
- - -
-
-
-

- 技术路线图 -

-

持续完善的数据处理生态系统

-
-
-
-
-
-
-
-
规划中
-
- -
-

数据迁移与同步工具

-

- 企业级数据迁移平台，支持多种数据库和存储系统之间的数据传输， - 提供实时同步、增量更新和数据转换功能。 -

-
- -
-
计划特性：
-
    -
  • 跨平台数据迁移（MySQL/PostgreSQL/MongoDB）
    • -
  • 实时数据同步和CDC支持
    • -
  • 数据格式转换和映射配置
    • -
  • 断点续传和错误恢复
    • -
  • 可视化监控和性能优化
    • -
-
- -
- -
-
-
- -
-
-
-
规划中
-
- -
-

数据质量监控平台

-

- 智能数据质量评估系统，自动检测数据完整性、一致性和准确性问题， - 提供实时监控、告警通知和质量报告生成。 -

-
- -
-
计划特性：
-
    -
  • 智能数据质量评分算法
    • -
  • 异常数据自动检测和标记
    • -
  • 实时监控Dashboard和告警
    • -
  • 质量趋势分析和预测
    • -
  • 自动化数据修复建议
    • -
-
- -
- -
-
-
-