From 42ff7c9f7c8746f659692c4446532a05c851b47c Mon Sep 17 00:00:00 2001 From: andersonid Date: Fri, 17 Oct 2025 10:05:57 -0300 Subject: [PATCH] =?UTF-8?q?Feature:=20Storage=20Analysis=20-=20nova=20se?= =?UTF-8?q?=C3=A7=C3=A3o=20para=20an=C3=A1lise=20de=20storage=20com=20m?= =?UTF-8?q?=C3=A9tricas,=20gr=C3=A1ficos=20e=20tabelas=20detalhadas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/routes.py | 145 +++++++++ app/core/kubernetes_client.py | 28 ++ app/static/index.html | 536 +++++++++++++++++++++++++++++++++- 3 files changed, 706 insertions(+), 3 deletions(-) diff --git a/app/api/routes.py b/app/api/routes.py index ae680a0..1b6c424 100644 --- a/app/api/routes.py +++ b/app/api/routes.py @@ -2404,3 +2404,148 @@ async def get_hybrid_health(): except Exception as e: logger.error(f"Error checking hybrid health: {e}") raise HTTPException(status_code=500, detail=str(e)) + +@api_router.get("/storage/analysis") +async def get_storage_analysis(k8s_client=Depends(get_k8s_client)): + """ + Get comprehensive storage analysis including PVCs, storage classes, and usage patterns. + """ + try: + logger.info("Starting storage analysis...") + + # Get all PVCs + pvcs = await k8s_client.get_all_pvcs() + logger.info(f"Found {len(pvcs)} PVCs") + + # Get storage classes + storage_classes = await k8s_client.get_storage_classes() + logger.info(f"Found {len(storage_classes)} storage classes") + + # Analyze storage usage by namespace + namespace_storage = {} + total_storage_used = 0 + storage_warnings = 0 + + for pvc in pvcs: + namespace = pvc.metadata.namespace + if namespace not in namespace_storage: + namespace_storage[namespace] = { + 'namespace': namespace, + 'storage_used': 0, + 'pvc_count': 0, + 'storage_classes': set(), + 'utilization_percent': 0 + } + + # Calculate storage used (convert to bytes) + storage_size = pvc.spec.resources.requests.get('storage', '0') + storage_bytes = _parse_storage_size(storage_size) + + namespace_storage[namespace]['storage_used'] += storage_bytes + namespace_storage[namespace]['pvc_count'] += 1 + namespace_storage[namespace]['storage_classes'].add(pvc.spec.storage_class_name or 'default') + + total_storage_used += storage_bytes + + # Check for storage warnings (PVCs with no storage class, etc.) + if not pvc.spec.storage_class_name: + storage_warnings += 1 + + # Calculate utilization percentages (mock for now) + for ns_data in namespace_storage.values(): + # Mock utilization calculation - in real implementation, this would come from Prometheus + ns_data['utilization_percent'] = min(100, (ns_data['storage_used'] / (1024**4)) * 10) # Mock calculation + ns_data['storage_classes'] = list(ns_data['storage_classes']) + + # Get top storage workloads + top_storage_workloads = [] + for ns_data in namespace_storage.values(): + if ns_data['storage_used'] > 0: + top_storage_workloads.append({ + 'name': ns_data['namespace'], + 'namespace': ns_data['namespace'], + 'storage_used': ns_data['storage_used'], + 'pvc_count': ns_data['pvc_count'] + }) + + # Sort by storage usage and take top 10 + top_storage_workloads.sort(key=lambda x: x['storage_used'], reverse=True) + top_storage_workloads = top_storage_workloads[:10] + + # Calculate max storage for percentage calculations + max_storage = max([w['storage_used'] for w in top_storage_workloads], default=1) + + # Analyze storage classes + storage_class_analysis = {} + for pvc in pvcs: + sc_name = pvc.spec.storage_class_name or 'default' + if sc_name not in storage_class_analysis: + storage_class_analysis[sc_name] = { + 'name': sc_name, + 'pvc_count': 0, + 'total_storage': 0 + } + + storage_class_analysis[sc_name]['pvc_count'] += 1 + storage_size = pvc.spec.resources.requests.get('storage', '0') + storage_bytes = _parse_storage_size(storage_size) + storage_class_analysis[sc_name]['total_storage'] += storage_bytes + + # Convert to list and sort by PVC count + storage_classes_list = list(storage_class_analysis.values()) + storage_classes_list.sort(key=lambda x: x['pvc_count'], reverse=True) + + # Calculate overall storage utilization + storage_utilization_percent = min(100, (total_storage_used / (1024**4)) * 5) # Mock calculation + + return { + "total_pvcs": len(pvcs), + "total_storage_used": total_storage_used, + "storage_utilization_percent": round(storage_utilization_percent, 1), + "storage_warnings": storage_warnings, + "namespace_storage": list(namespace_storage.values()), + "top_storage_workloads": top_storage_workloads, + "storage_classes": storage_classes_list, + "max_storage": max_storage, + "timestamp": datetime.now().isoformat() + } + + except Exception as e: + logger.error(f"Error in storage analysis: {e}") + raise HTTPException(status_code=500, detail=str(e)) + +def _parse_storage_size(size_str: str) -> int: + """ + Parse storage size string (e.g., '10Gi', '100Mi') to bytes. + """ + if not size_str or size_str == '0': + return 0 + + size_str = size_str.upper() + + # Extract number and unit + import re + match = re.match(r'^(\d+(?:\.\d+)?)\s*([A-Z]+)$', size_str) + if not match: + return 0 + + number = float(match.group(1)) + unit = match.group(2) + + # Convert to bytes + multipliers = { + 'B': 1, + 'KB': 1024, + 'MB': 1024**2, + 'GB': 1024**3, + 'TB': 1024**4, + 'PB': 1024**5, + 'KIB': 1024, + 'MIB': 1024**2, + 'GIB': 1024**3, + 'TIB': 1024**4, + 'PIB': 1024**5 + } + + multiplier = multipliers.get(unit, 1) + return int(number * multiplier) diff --git a/app/core/kubernetes_client.py b/app/core/kubernetes_client.py index 7bb67a6..90ac592 100644 --- a/app/core/kubernetes_client.py +++ b/app/core/kubernetes_client.py @@ -530,3 +530,31 @@ class K8sClient: except ApiException as e: logger.error(f"Error collecting node information: {e}") raise + + async def get_all_pvcs(self) -> List[Any]: + """Get all PersistentVolumeClaims in the cluster""" + if not self.initialized: + raise RuntimeError("Kubernetes client not initialized") + + try: + # List all PVCs in all namespaces + pvcs = self.v1.list_persistent_volume_claim_for_all_namespaces(watch=False) + return pvcs.items + + except ApiException as e: + logger.error(f"Error getting PVCs: {e}") + raise + + async def get_storage_classes(self) -> List[Any]: + """Get all StorageClasses in the cluster""" + if not self.initialized: + raise RuntimeError("Kubernetes client not initialized") + + try: + # List all storage classes + storage_classes = self.v1.list_storage_class(watch=False) + return storage_classes.items + + except ApiException as e: + logger.error(f"Error getting storage classes: {e}") + raise diff --git a/app/static/index.html b/app/static/index.html index f4a0ccb..d52263e 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -1699,6 +1699,12 @@ Requests & Limits +
  • + + + Storage Analysis + +
    • @@ -2027,7 +2033,139 @@ - + +
    +
    +

    Storage Analysis

    +

    Analyze storage usage, consumption patterns, and available capacity across workloads

    +
    + + +
    +
    +
    + +
    +
    +
    -
    +
    Total PVCs
    +
    +
    +
    +
    + +
    +
    +
    -
    +
    Storage Used
    +
    +
    +
    +
    + +
    +
    +
    -
    +
    Utilization
    +
    +
    +
    +
    + +
    +
    +
    -
    +
    Storage Warnings
    +
    +
    +
    + + +
    +
    +

    Storage Analytics

    + +
    +
    +
    +
    +

    + + Storage Usage Trend (24h) +

    +
    + +
    +
    +
    +

    + + Storage by Namespace +

    +
    + +
    +
    +
    +
    +
    +

    + + Top 10 Workloads by Storage Usage +

    +
    + +
    +
    +
    +

    + + Storage Classes Distribution +

    +
    + +
    +
    +
    +
    +
    + + +
    +
    +

    Storage Details by Namespace

    +
    +
    + + +
    +
    +
    +
    + +
    +
    +
    + +

    Historical Analysis

    @@ -2200,6 +2338,395 @@ totalSteps: 3 }; + // Storage Analysis Functions + async function loadStorageAnalysis() { + try { + console.log('Loading storage analysis...'); + showLoading('storage-details-container'); + + // Load storage data from API + const response = await fetch('/api/v1/storage/analysis'); + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const data = await response.json(); + + // Update storage metrics cards + updateStorageMetrics(data); + + // Load storage charts + await loadStorageCharts(data); + + // Update storage details table + updateStorageDetails(data); + + console.log('Storage analysis loaded successfully'); + + } catch (error) { + console.error('Error loading storage analysis:', error); + showError('storage-details-container', `Error loading storage analysis: ${error.message}`); + } + } + + function updateStorageMetrics(data) { + // Update storage overview cards + document.getElementById('total-pvcs').textContent = data.total_pvcs || '0'; + document.getElementById('total-storage-used').textContent = formatBytes(data.total_storage_used || 0); + document.getElementById('storage-utilization').textContent = `${data.storage_utilization_percent || 0}%`; + document.getElementById('storage-warnings').textContent = data.storage_warnings || '0'; + } + + async function loadStorageCharts(data) { + try { + // Storage Usage Trend Chart + await loadStorageTrendChart(data); + + // Storage by Namespace Chart + await loadStorageNamespaceChart(data); + + // Top Storage Workloads Chart + await loadTopStorageWorkloadsChart(data); + + // Storage Classes Chart + await loadStorageClassesChart(data); + + } catch (error) { + console.error('Error loading storage charts:', error); + } + } + + async function loadStorageTrendChart(data) { + const container = document.getElementById('storage-trend-chart'); + + // Simulate trend data (in real implementation, this would come from Prometheus) + const trendData = generateStorageTrendData(); + + container.innerHTML = ` +
    + +
    + `; + + // Create line chart using Chart.js (if available) or Victory.js + setTimeout(() => { + if (typeof Victory !== 'undefined') { + const chart = new Victory.Line({ + data: trendData, + width: 400, + height: 300, + style: { + data: { stroke: "#0066cc" } + } + }); + chart.render(container.querySelector('#storage-trend-canvas')); + } else { + // Fallback to simple visualization + container.innerHTML = ` +
    + +

    Storage trend visualization

    +

    + Chart library not available +

    +
    + `; + } + }, 100); + } + + async function loadStorageNamespaceChart(data) { + const container = document.getElementById('storage-namespace-chart'); + + // Create horizontal bar chart for namespace storage distribution + const namespaceData = data.namespace_storage || []; + const totalStorage = namespaceData.reduce((sum, ns) => sum + (ns.storage_used || 0), 0); + + if (namespaceData.length === 0) { + container.innerHTML = ` +
    + +

    No storage data available

    +
    + `; + return; + } + + // Sort by storage usage and take top 10 + const sortedData = namespaceData + .sort((a, b) => (b.storage_used || 0) - (a.storage_used || 0)) + .slice(0, 10); + + const chartHTML = sortedData.map((ns, index) => { + const percentage = totalStorage > 0 ? ((ns.storage_used || 0) / totalStorage * 100) : 0; + const width = Math.max(percentage * 3, 2); // Minimum 2px width + + return ` +
    +
    + ${ns.namespace} +
    +
    +
    +
    ${percentage.toFixed(1)}%
    +
    +
    + ${formatBytes(ns.storage_used || 0)} used +
    +
    + `; + }).join(''); + + container.innerHTML = ` +
    + ${chartHTML} +
    + `; + } + + async function loadTopStorageWorkloadsChart(data) { + const container = document.getElementById('top-storage-workloads-chart'); + + const workloadData = data.top_storage_workloads || []; + + if (workloadData.length === 0) { + container.innerHTML = ` +
    + +

    No workload storage data available

    +
    + `; + return; + } + + const chartHTML = workloadData.map((workload, index) => { + const percentage = data.max_storage > 0 ? (workload.storage_used / data.max_storage * 100) : 0; + const width = Math.max(percentage * 2, 2); + + return ` +
    +
    +
    + ${workload.name} +
    +
    + ${formatBytes(workload.storage_used)} +
    +
    +
    +
    +
    +
    + Namespace: ${workload.namespace} • PVCs: ${workload.pvc_count} +
    +
    + `; + }).join(''); + + container.innerHTML = ` +
    + ${chartHTML} +
    + `; + } + + async function loadStorageClassesChart(data) { + const container = document.getElementById('storage-classes-chart'); + + const storageClassData = data.storage_classes || []; + + if (storageClassData.length === 0) { + container.innerHTML = ` +
    + +

    No storage class data available

    +
    + `; + return; + } + + const chartHTML = storageClassData.map((sc, index) => { + const percentage = data.total_pvcs > 0 ? (sc.pvc_count / data.total_pvcs * 100) : 0; + const width = Math.max(percentage * 2, 2); + + return ` +
    +
    +
    + ${sc.name} +
    +
    + ${sc.pvc_count} PVCs +
    +
    +
    +
    +
    ${percentage.toFixed(1)}%
    +
    +
    + `; + }).join(''); + + container.innerHTML = ` +
    + ${chartHTML} +
    + `; + } + + function updateStorageDetails(data) { + const container = document.getElementById('storage-details-container'); + + const namespaceData = data.namespace_storage || []; + + if (namespaceData.length === 0) { + container.innerHTML = ` +
    + +

    No storage data available

    +
    + `; + return; + } + + // Sort by storage usage + const sortedData = namespaceData.sort((a, b) => (b.storage_used || 0) - (a.storage_used || 0)); + + const tableHTML = ` +
    + + + + + + + + + + + + + ${sortedData.map(ns => ` + + + + + + + + + `).join('')} + +
    NamespaceStorage UsedPVCsStorage ClassesUtilizationStatus
    +
    + ${ns.namespace} +
    +     		+
    + ${formatBytes(ns.storage_used || 0)} +
    +     		+ ${ns.pvc_count || 0} + +
    + ${(ns.storage_classes || []).map(sc => + `${sc}` + ).join(' ')} +
    +     		+
    +
    + ${ns.utilization_percent || 0}% +
    +     		+ + ${getStorageStatusText(ns)} + +
    +
    + `; + + container.innerHTML = tableHTML; + } + + function generateStorageTrendData() { + // Generate mock trend data for 24 hours + const data = []; + const now = new Date(); + + for (let i = 23; i >= 0; i--) { + const time = new Date(now.getTime() - (i * 60 * 60 * 1000)); + const usage = 100 + Math.random() * 50; // Mock usage between 100-150GB + + data.push({ + x: time.toISOString(), + y: usage + }); + } + + return data; + } + + function formatBytes(bytes) { + if (bytes === 0) return '0 B'; + + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + + return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; + } + + function getStorageStatusClass(ns) { + const utilization = ns.utilization_percent || 0; + if (utilization >= 90) return 'danger'; + if (utilization >= 75) return 'warning'; + if (utilization >= 50) return 'info'; + return 'success'; + } + + function getStorageStatusText(ns) { + const utilization = ns.utilization_percent || 0; + if (utilization >= 90) return 'Critical'; + if (utilization >= 75) return 'Warning'; + if (utilization >= 50) return 'Info'; + return 'Healthy'; + } + // Initialize the application document.addEventListener('DOMContentLoaded', function() { initializeApp(); @@ -2408,6 +2935,7 @@ // Refresh buttons document.getElementById('refresh-workloads').addEventListener('click', loadRequestsLimits); + document.getElementById('refresh-storage').addEventListener('click', loadStorageAnalysis); document.getElementById('refresh-historical').addEventListener('click', loadHistoricalAnalysis); // Close dropdown when clicking outside @@ -2446,10 +2974,12 @@ loadWorkloadScanner(false); // Use cache if available } else if (section === 'requests-limits') { loadRequestsLimits(); + } else if (section === 'storage-analysis') { + loadStorageAnalysis(); } else if (section === 'vpa-management') { - loadVPAManagement(); + loadVPAManagement(); } else if (section === 'historical-analysis') { - loadHistoricalAnalysis(); + loadHistoricalAnalysis(); } else if (section === 'settings') { loadSettings(); }