完善页面

This commit is contained in:
2025-08-11 09:34:29 +08:00
parent 0ac375eb50
commit d42cefd9ca
4 changed files with 306 additions and 165 deletions

13
app.py
View File

@@ -76,6 +76,15 @@ if __name__ == '__main__':
logger.info("=== BigDataTool 启动 ===")
logger.info("应用架构:模块化")
logger.info("支持功能:单表查询、分表查询、多主键查询、配置管理、查询历史")
logger.info("访问地址http://localhost:5000")
# 从环境变量获取配置支持Docker部署
import os
host = os.getenv('FLASK_HOST', '0.0.0.0')
port = int(os.getenv('FLASK_PORT', 5000))
debug = os.getenv('FLASK_DEBUG', 'True').lower() == 'true'
logger.info(f"访问地址http://{host}:{port}")
logger.info("API文档/api/* 路径下的所有端点")
app.run(debug=True, port=5000)
logger.info(f"配置信息 - 主机: {host}, 端口: {port}, 调试: {debug}")
app.run(debug=debug, host=host, port=port)

View File

@@ -68,6 +68,32 @@ def setup_routes(app, query_log_collector):
return render_template('redis_test.html')
# 基础API
@app.route('/api/health')
def health_check():
"""健康检查端点用于Docker健康检查和服务监控"""
try:
# 检查应用基本状态
current_time = datetime.now().isoformat()
# 简单的数据库连接检查(可选)
from .database import ensure_database
db_status = ensure_database()
return jsonify({
'status': 'healthy',
'timestamp': current_time,
'service': 'BigDataTool',
'version': '2.0',
'database': 'ok' if db_status else 'warning'
})
except Exception as e:
logger.error(f"健康检查失败: {str(e)}")
return jsonify({
'status': 'unhealthy',
'error': str(e),
'timestamp': datetime.now().isoformat()
}), 503
@app.route('/api/default-config')
def get_default_config():
return jsonify(DEFAULT_CONFIG)

View File

@@ -908,35 +908,65 @@ function displayStats(results) {
function displayDifferences() {
const differencesContainer = document.getElementById('differences');
// 保存当前搜索框的值
const currentSearchValue = document.getElementById('differenceSearch')?.value || '';
if (!filteredDifferenceResults.length) {
differencesContainer.innerHTML = '<p class="text-success"><i class="fas fa-check"></i> 未发现差异</p>';
return;
}
// 计算分页
const totalPages = Math.ceil(filteredDifferenceResults.length / differencePageSize);
// 按主键分组差异
const groupedDifferences = groupDifferencesByKey(filteredDifferenceResults);
const totalGroups = Object.keys(groupedDifferences).length;
// 计算分页(基于主键组)
const groupKeys = Object.keys(groupedDifferences);
const totalPages = Math.ceil(groupKeys.length / differencePageSize);
const startIndex = (currentDifferencePage - 1) * differencePageSize;
const endIndex = startIndex + differencePageSize;
const currentPageData = filteredDifferenceResults.slice(startIndex, endIndex);
const currentPageKeys = groupKeys.slice(startIndex, endIndex);
// 统计字段差异类型
const fieldStats = {};
filteredDifferenceResults.forEach(diff => {
if (diff.field) {
fieldStats[diff.field] = (fieldStats[diff.field] || 0) + 1;
}
});
let html = `
<!-- 字段筛选按钮 -->
<div class="mb-3">
<div class="btn-group btn-group-sm flex-wrap" role="group">
<button type="button" class="btn btn-outline-primary active" onclick="filterByField('')">
全部 (${filteredDifferenceResults.length})
</button>
${Object.entries(fieldStats).map(([field, count]) => `
<button type="button" class="btn btn-outline-secondary" onclick="filterByField('${field}')">
${field} (${count})
</button>
`).join('')}
</div>
</div>
<!-- 分页控制 -->
<div class="row mb-3">
<div class="col-md-6">
<div class="d-flex align-items-center">
<label class="form-label me-2 mb-0">每页显示:</label>
<select class="form-select form-select-sm me-2" style="width: auto;" onchange="changeDifferencePageSize(this.value)">
<option value="5" ${differencePageSize == 5 ? 'selected' : ''}>5</option>
<option value="10" ${differencePageSize == 10 ? 'selected' : ''}>10</option>
<option value="20" ${differencePageSize == 20 ? 'selected' : ''}>20</option>
<option value="50" ${differencePageSize == 50 ? 'selected' : ''}>50</option>
<option value="100" ${differencePageSize == 100 ? 'selected' : ''}>100</option>
<option value="5" ${differencePageSize == 5 ? 'selected' : ''}>5</option>
<option value="10" ${differencePageSize == 10 ? 'selected' : ''}>10</option>
<option value="20" ${differencePageSize == 20 ? 'selected' : ''}>20</option>
<option value="50" ${differencePageSize == 50 ? 'selected' : ''}>50</option>
<option value="100" ${differencePageSize == 100 ? 'selected' : ''}>100</option>
<option value="custom_diff">自定义</option>
</select>
<input type="number" class="form-control form-control-sm me-2" style="width: 80px; display: none;"
id="customDiffPageSize" placeholder="数量" min="1" max="1000"
onchange="setCustomDifferencePageSize(this.value)" onkeypress="handleCustomDiffPageSizeEnter(event)">
<span class="ms-2 text-muted">共 ${filteredDifferenceResults.length} 条差异记录</span>
<span class="ms-2 text-muted">共 ${totalGroups} 个主键组,${filteredDifferenceResults.length} 条差异记录</span>
</div>
</div>
<div class="col-md-6">
@@ -951,94 +981,111 @@ function displayDifferences() {
<div class="input-group">
<span class="input-group-text"><i class="fas fa-search"></i></span>
<input type="text" class="form-control" placeholder="搜索主键、字段名或值内容..."
onkeyup="searchDifferenceResults(this.value)" id="differenceSearch">
onkeypress="if(event.key === 'Enter') searchDifferenceResults(this.value)"
id="differenceSearch"
value="${currentSearchValue}">
<button class="btn btn-outline-secondary" type="button" onclick="searchDifferenceResults(document.getElementById('differenceSearch').value)">
<i class="fas fa-search"></i> 搜索
</button>
<button class="btn btn-outline-secondary" type="button" onclick="clearDifferenceSearch()">
<i class="fas fa-times"></i> 清除
</button>
</div>
</div>
`;
// 显示当前页数据
currentPageData.forEach((diff, index) => {
const globalIndex = startIndex + index + 1;
if (diff.message) {
// 记录不存在的情况
html += `
<div class="difference-item card mb-3 border-warning">
<div class="card-header bg-light">
<div class="row align-items-center">
<div class="col">
<strong>差异 #${globalIndex}</strong>
<span class="badge bg-warning ms-2">记录缺失</span>
</div>
<div class="col-auto">
<button class="btn btn-sm btn-outline-info" onclick='showDifferenceRawData(${JSON.stringify(JSON.stringify(diff.key))})'>
<i class="fas fa-code"></i> 原生数据
</button>
</div>
// 显示当前页的主键组
currentPageKeys.forEach((key, groupIndex) => {
const diffs = groupedDifferences[key];
const globalIndex = startIndex + groupIndex + 1;
html += `
<div class="card mb-3 border-primary">
<div class="card-header bg-primary bg-opacity-10">
<div class="row align-items-center">
<div class="col">
<strong>主键组 #${globalIndex}</strong>
<span class="badge bg-primary ms-2">${diffs.length} 个差异字段</span>
</div>
<div class="col-auto">
<button class="btn btn-sm btn-outline-secondary" type="button"
data-bs-toggle="collapse" data-bs-target="#keyGroup${globalIndex}">
<i class="fas fa-chevron-down"></i> 展开/收起
</button>
<button class="btn btn-sm btn-outline-info ms-2" onclick='showDifferenceRawData(${JSON.stringify(JSON.stringify(key))})'>
<i class="fas fa-code"></i> 原生数据
</button>
</div>
<p class="mb-0 mt-2"><strong>主键:</strong> ${formatCompositeKey(diff.key)}</p>
</div>
<div class="card-body">
<p class="text-warning"><i class="fas fa-exclamation-triangle"></i> ${diff.message}</p>
</div>
<p class="mb-0 mt-2"><strong>主键:</strong> ${formatCompositeKey(key)}</p>
</div>
`;
} else {
// 字段值差异的情况
const isJson = diff.is_json;
const isArray = diff.is_array;
const jsonClass = isJson ? 'json-field' : '';
html += `
<div class="difference-item card mb-3 border-danger">
<div class="card-header bg-light">
<div class="row align-items-center">
<div class="col">
<strong>差异 #${globalIndex}</strong>
<span class="badge bg-danger ms-2">字段差异</span>
${isJson ? '<span class="badge bg-info ms-2">JSON字段</span>' : ''}
${isArray ? '<span class="badge bg-warning ms-2">数组字段</span>' : ''}
</div>
<div class="col-auto">
<button class="btn btn-sm btn-outline-secondary me-2" type="button"
data-bs-toggle="collapse" data-bs-target="#diffCollapse${globalIndex}">
<i class="fas fa-eye"></i> 查看详情
</button>
<button class="btn btn-sm btn-outline-info" onclick='showDifferenceRawData(${JSON.stringify(JSON.stringify(diff.key))})'>
<i class="fas fa-code"></i> 原生数据
</button>
</div>
</div>
<p class="mb-0 mt-2"><strong>主键:</strong> ${formatCompositeKey(diff.key)}</p>
<p class="mb-0"><strong>差异字段:</strong> ${diff.field}</p>
<div class="collapse show" id="keyGroup${globalIndex}">
<div class="card-body">
`;
// 显示该主键下的所有差异字段
diffs.forEach((diff, diffIndex) => {
if (diff.message) {
// 记录不存在的情况
html += `
<div class="alert alert-warning mb-2">
<i class="fas fa-exclamation-triangle"></i> ${diff.message}
</div>
<div class="collapse" id="diffCollapse${globalIndex}">
<div class="card-body">
<div class="mb-4">
<div class="field-header mb-2">
<i class="fas fa-tag text-primary"></i>
<strong>${diff.field}</strong>
`;
} else {
// 字段值差异的情况
const isJson = diff.is_json;
const isArray = diff.is_array;
const jsonClass = isJson ? 'json-field' : '';
const fieldId = `field_${globalIndex}_${diffIndex}`;
html += `
<div class="mb-3 border-start border-3 border-danger ps-3">
<div class="d-flex justify-content-between align-items-start mb-2">
<div>
<strong class="text-danger">${diff.field}</strong>
${isJson ? '<span class="badge bg-info ms-2">JSON</span>' : ''}
${isArray ? '<span class="badge bg-warning ms-2">数组</span>' : ''}
</div>
<button class="btn btn-sm btn-outline-secondary" type="button"
data-bs-toggle="collapse" data-bs-target="#${fieldId}">
<i class="fas fa-eye"></i> 查看
</button>
</div>
<div class="collapse" id="${fieldId}">
<div class="row">
<div class="col-md-6">
<div class="card">
<div class="card-header bg-success bg-opacity-10">
<small class="text-success fw-bold">生产环境</small>
</div>
<div class="card-body p-2">
<pre class="field-value mb-0 ${jsonClass}" style="max-height: 200px; overflow-y: auto;">${escapeHtml(diff.pro_value)}</pre>
</div>
</div>
</div>
<!-- 环境对比数据行 -->
<div class="row">
<div class="col-12">
<div class="field-container position-relative">
<pre class="field-value bg-light p-2 rounded mb-0 ${jsonClass}" style="max-height: 400px; overflow-y: auto; margin: 0;">${escapeHtml(diff.pro_value)}
${escapeHtml(diff.test_value)}</pre>
<button class="btn btn-sm btn-outline-secondary copy-btn"
onclick="copyToClipboard('${escapeForJs(diff.pro_value)}\n${escapeForJs(diff.test_value)}', this)"
title="复制全部内容">
<i class="fas fa-copy"></i>
</button>
<div class="col-md-6">
<div class="card">
<div class="card-header bg-info bg-opacity-10">
<small class="text-info fw-bold">测试环境</small>
</div>
<div class="card-body p-2">
<pre class="field-value mb-0 ${jsonClass}" style="max-height: 200px; overflow-y: auto;">${escapeHtml(diff.test_value)}</pre>
</div>
</div>
</div>
</div>
</div>
</div>
`;
}
});
html += `
</div>
</div>
`;
}
</div>
`;
});
// 底部分页
@@ -1055,6 +1102,15 @@ ${escapeHtml(diff.test_value)}</pre>
}
differencesContainer.innerHTML = html;
// 恢复搜索框的焦点和光标位置
if (currentSearchValue) {
const searchInput = document.getElementById('differenceSearch');
if (searchInput) {
searchInput.focus();
searchInput.setSelectionRange(searchInput.value.length, searchInput.value.length);
}
}
}
// HTML转义函数防止XSS
@@ -1068,6 +1124,9 @@ function escapeHtml(text) {
function displayIdenticalResults() {
const identicalContainer = document.getElementById('identical-results');
// 保存当前搜索框的值
const currentSearchValue = document.getElementById('identicalSearch')?.value || '';
if (!filteredIdenticalResults.length) {
identicalContainer.innerHTML = '<p class="text-muted"><i class="fas fa-info-circle"></i> 没有完全相同的记录</p>';
return;
@@ -1111,7 +1170,15 @@ function displayIdenticalResults() {
<div class="input-group">
<span class="input-group-text"><i class="fas fa-search"></i></span>
<input type="text" class="form-control" placeholder="搜索主键或字段内容..."
onkeyup="searchIdenticalResults(this.value)" id="identicalSearch">
onkeypress="if(event.key === 'Enter') searchIdenticalResults(this.value)"
id="identicalSearch"
value="${currentIdenticalSearchTerm || ''}">
<button class="btn btn-outline-secondary" type="button" onclick="searchIdenticalResults(document.getElementById('identicalSearch').value)">
<i class="fas fa-search"></i> 搜索
</button>
<button class="btn btn-outline-secondary" type="button" onclick="clearIdenticalSearch()">
<i class="fas fa-times"></i> 清除
</button>
</div>
</div>
`;
@@ -1219,6 +1286,15 @@ ${escapeHtml(String(testValue))}</pre>
}
identicalContainer.innerHTML = html;
// 恢复搜索框的焦点和光标位置
if (currentSearchValue) {
const searchInput = document.getElementById('identicalSearch');
if (searchInput) {
searchInput.focus();
searchInput.setSelectionRange(searchInput.value.length, searchInput.value.length);
}
}
}
// 生成分页导航
@@ -1334,9 +1410,14 @@ function handleCustomPageSizeEnter(event) {
}
// 搜索相同结果
let currentIdenticalSearchTerm = '';
function searchIdenticalResults(searchTerm) {
if (!currentResults) return;
// 保存搜索词
currentIdenticalSearchTerm = searchTerm;
if (!searchTerm.trim()) {
filteredIdenticalResults = currentResults.identical_results;
} else {
@@ -1361,6 +1442,15 @@ function searchIdenticalResults(searchTerm) {
displayIdenticalResults();
}
// 清除相同记录搜索
function clearIdenticalSearch() {
currentIdenticalSearchTerm = '';
document.getElementById('identicalSearch').value = '';
filteredIdenticalResults = currentResults.identical_results;
currentIdenticalPage = 1;
displayIdenticalResults();
}
// 生成差异分页导航
function generateDifferencePagination(currentPage, totalPages) {
if (totalPages <= 1) return '';
@@ -1474,14 +1564,28 @@ function handleCustomDiffPageSizeEnter(event) {
}
// 搜索差异结果
let currentSearchTerm = '';
function searchDifferenceResults(searchTerm) {
if (!currentResults) return;
// 保存搜索词
currentSearchTerm = searchTerm;
if (!searchTerm.trim()) {
filteredDifferenceResults = currentResults.differences;
// 如果有字段筛选,应用字段筛选;否则显示全部
if (currentFieldFilter) {
filteredDifferenceResults = currentResults.differences.filter(diff => diff.field === currentFieldFilter);
} else {
filteredDifferenceResults = currentResults.differences;
}
} else {
const term = searchTerm.toLowerCase();
filteredDifferenceResults = currentResults.differences.filter(diff => {
let baseResults = currentFieldFilter ?
currentResults.differences.filter(diff => diff.field === currentFieldFilter) :
currentResults.differences;
filteredDifferenceResults = baseResults.filter(diff => {
// 搜索主键
const keyStr = JSON.stringify(diff.key).toLowerCase();
if (keyStr.includes(term)) return true;
@@ -1504,6 +1608,22 @@ function searchDifferenceResults(searchTerm) {
displayDifferences();
}
// 清除差异搜索
function clearDifferenceSearch() {
currentSearchTerm = '';
document.getElementById('differenceSearch').value = '';
// 重新应用字段筛选
if (currentFieldFilter) {
filteredDifferenceResults = currentResults.differences.filter(diff => diff.field === currentFieldFilter);
} else {
filteredDifferenceResults = currentResults.differences;
}
currentDifferencePage = 1;
displayDifferences();
}
// 复制到剪贴板
function copyToClipboard(text, button) {
navigator.clipboard.writeText(text).then(function() {
@@ -4271,4 +4391,69 @@ async function batchDeleteQueryHistory() {
console.error('批量删除Cassandra查询历史记录失败:', error);
showAlert('danger', `批量删除失败: ${error.message}`);
}
}
// 按主键分组差异数据
function groupDifferencesByKey(differences) {
const grouped = {};
differences.forEach(diff => {
const keyStr = typeof diff.key === 'object' ? JSON.stringify(diff.key) : diff.key;
if (!grouped[keyStr]) {
grouped[keyStr] = [];
}
grouped[keyStr].push(diff);
});
return grouped;
}
// 按字段筛选差异
let currentFieldFilter = '';
function filterByField(field) {
currentFieldFilter = field;
// 更新按钮状态
const buttons = document.querySelectorAll('.btn-group button');
buttons.forEach(btn => {
btn.classList.remove('active', 'btn-outline-primary');
btn.classList.add('btn-outline-secondary');
});
// 设置当前选中的按钮
if (field === '') {
buttons[0].classList.remove('btn-outline-secondary');
buttons[0].classList.add('btn-outline-primary', 'active');
} else {
buttons.forEach(btn => {
if (btn.textContent.includes(field + ' (')) {
btn.classList.remove('btn-outline-secondary');
btn.classList.add('btn-outline-primary', 'active');
}
});
}
// 筛选差异数据(考虑搜索条件)
let baseResults = field === '' ?
currentResults.differences :
currentResults.differences.filter(diff => diff.field === field);
// 如果有搜索条件,应用搜索过滤
if (currentSearchTerm && currentSearchTerm.trim()) {
const term = currentSearchTerm.toLowerCase();
filteredDifferenceResults = baseResults.filter(diff => {
const keyStr = JSON.stringify(diff.key).toLowerCase();
if (keyStr.includes(term)) return true;
if (diff.field && diff.field.toLowerCase().includes(term)) return true;
if (diff.pro_value && String(diff.pro_value).toLowerCase().includes(term)) return true;
if (diff.test_value && String(diff.test_value).toLowerCase().includes(term)) return true;
if (diff.message && diff.message.toLowerCase().includes(term)) return true;
return false;
});
} else {
filteredDifferenceResults = baseResults;
}
// 重置页码并重新显示
currentDifferencePage = 1;
displayDifferences();
}

View File

@@ -295,85 +295,6 @@
</div>
</div>
</div>
<!-- 未来工具规划 -->
<div class="row mt-5">
<div class="col-12">
<div class="text-center mb-4">
<h3 class="text-muted">
<i class="fas fa-road"></i> 技术路线图
</h3>
<p class="text-muted">持续完善的数据处理生态系统</p>
</div>
</div>
</div>
<div class="row mt-4">
<div class="col-lg-6 col-md-12">
<div class="tool-card">
<div class="text-center">
<div class="feature-badge coming-soon">规划中</div>
<div class="tool-icon">
<i class="fas fa-file-export"></i>
</div>
<h3 class="tool-title">数据迁移与同步工具</h3>
<p class="tool-description">
企业级数据迁移平台,支持多种数据库和存储系统之间的数据传输,
提供实时同步、增量更新和数据转换功能。
</p>
</div>
<div class="tool-features">
<h5><i class="fas fa-star text-warning"></i> 计划特性:</h5>
<ul>
<li><i class="fas fa-clock text-muted"></i> 跨平台数据迁移MySQL/PostgreSQL/MongoDB</li>
<li><i class="fas fa-clock text-muted"></i> 实时数据同步和CDC支持</li>
<li><i class="fas fa-clock text-muted"></i> 数据格式转换和映射配置</li>
<li><i class="fas fa-clock text-muted"></i> 断点续传和错误恢复</li>
<li><i class="fas fa-clock text-muted"></i> 可视化监控和性能优化</li>
</ul>
</div>
<div class="text-center">
<button class="tool-btn" disabled style="opacity: 0.6;">
<i class="fas fa-clock"></i> 开发中
</button>
</div>
</div>
</div>
<div class="col-lg-6 col-md-12">
<div class="tool-card">
<div class="text-center">
<div class="feature-badge coming-soon">规划中</div>
<div class="tool-icon">
<i class="fas fa-chart-line"></i>
</div>
<h3 class="tool-title">数据质量监控平台</h3>
<p class="tool-description">
智能数据质量评估系统,自动检测数据完整性、一致性和准确性问题,
提供实时监控、告警通知和质量报告生成。
</p>
</div>
<div class="tool-features">
<h5><i class="fas fa-star text-warning"></i> 计划特性:</h5>
<ul>
<li><i class="fas fa-clock text-muted"></i> 智能数据质量评分算法</li>
<li><i class="fas fa-clock text-muted"></i> 异常数据自动检测和标记</li>
<li><i class="fas fa-clock text-muted"></i> 实时监控Dashboard和告警</li>
<li><i class="fas fa-clock text-muted"></i> 质量趋势分析和预测</li>
<li><i class="fas fa-clock text-muted"></i> 自动化数据修复建议</li>
</ul>
</div>
<div class="text-center">
<button class="tool-btn" disabled style="opacity: 0.6;">
<i class="fas fa-clock"></i> 开发中
</button>
</div>
</div>
</div>
</div>
</div>
</div>