Compare commits
4 commits
Author | SHA1 | Date | |
---|---|---|---|
|
77fc6c21f6 | ||
|
e577184733 | ||
|
773b03a652 | ||
|
b33b2fbd6e |
5 changed files with 409 additions and 171 deletions
4
.gitignore
vendored
Normal file
4
.gitignore
vendored
Normal file
|
@ -0,0 +1,4 @@
|
||||||
|
2024/05/30.csv
|
||||||
|
2024/05/30-2.txt
|
||||||
|
results.csv
|
||||||
|
miner_logs.log
|
188
finder.py
188
finder.py
|
@ -2,8 +2,17 @@ import paramiko
|
||||||
import re
|
import re
|
||||||
import json
|
import json
|
||||||
import csv
|
import csv
|
||||||
|
import logging
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
|
# Constants for error types
|
||||||
|
ASIC_ERROR = "ASIC Error"
|
||||||
|
EEPROM_ERROR = "EEPROM Error"
|
||||||
|
CHIP_BIN_ERROR = "Chip Bin Error"
|
||||||
|
|
||||||
|
# Logging configuration
|
||||||
|
logging.basicConfig(filename='miner_logs.log', level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||||
|
|
||||||
# Load credentials from a JSON file
|
# Load credentials from a JSON file
|
||||||
def load_credentials(file_path):
|
def load_credentials(file_path):
|
||||||
with open(file_path, 'r') as file:
|
with open(file_path, 'r') as file:
|
||||||
|
@ -26,83 +35,92 @@ def read_ips(file_path):
|
||||||
ips = file.readlines()
|
ips = file.readlines()
|
||||||
return [ip.strip() for ip in ips]
|
return [ip.strip() for ip in ips]
|
||||||
|
|
||||||
# Function to check log files for keywords and ASIC errors
|
# Function to establish an SSH connection
|
||||||
def check_logs(ip, ssh_client, worker_id, current_date):
|
def establish_ssh_connection(ip, username, password):
|
||||||
logs = []
|
ssh_client = paramiko.SSHClient()
|
||||||
asic_errors = set() # Using set to avoid duplicate errors
|
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||||
results = [] # Using list to avoid duplicate entries
|
|
||||||
try:
|
try:
|
||||||
print(f"Checking logs on {ip}")
|
ssh_client.connect(ip, username=username, password=password, timeout=5)
|
||||||
stdin, stdout, stderr = ssh_client.exec_command("find /var/log/ -type f")
|
logging.info(f"Connected to {ip} with {username}")
|
||||||
log_files = stdout.readlines()
|
return ssh_client
|
||||||
for log_file in log_files:
|
|
||||||
log_file = log_file.strip()
|
|
||||||
print(f"Checking file: {log_file}") # Debug statement
|
|
||||||
|
|
||||||
# Read the log file content directly
|
|
||||||
stdin, stdout, stderr = ssh_client.exec_command(f"cat {log_file}")
|
|
||||||
log_content = stdout.read().decode('utf-8', errors='ignore')
|
|
||||||
print(f"Content of {log_file}: {log_content[:500]}") # Debug statement to show part of the log content
|
|
||||||
|
|
||||||
# Track unique errors within this log file
|
|
||||||
seen_errors = set()
|
|
||||||
for keyword, error_type in error_keywords.items():
|
|
||||||
if keyword in log_content and (log_file, error_type, keyword) not in seen_errors:
|
|
||||||
print(f"Found keyword '{keyword}' in {log_file}") # Debug statement
|
|
||||||
logs.append((log_file, error_type, keyword))
|
|
||||||
seen_errors.add((log_file, error_type, keyword))
|
|
||||||
|
|
||||||
# Check for ASIC chip errors and power-off messages
|
|
||||||
for match in asic_pattern.finditer(log_content):
|
|
||||||
chain, asic_count = match.groups()
|
|
||||||
asic_count = int(asic_count)
|
|
||||||
asic_errors.add((chain, asic_count))
|
|
||||||
print(f"Chain {chain} has {asic_count} chips.") # Debug statement
|
|
||||||
|
|
||||||
# Check for power-off messages
|
|
||||||
for match in power_off_pattern.finditer(log_content):
|
|
||||||
chain, found_asic_count, board = match.groups()
|
|
||||||
found_asic_count = int(found_asic_count)
|
|
||||||
chain = int(chain)
|
|
||||||
if (log_file, "ASIC Error", f"Chain {chain} has failed with {found_asic_count} ASICs found and will power off hash board {board}") not in seen_errors:
|
|
||||||
results.append((current_date, worker_id, ip, log_file, "ASIC Error", f"Chain {chain} has failed with {found_asic_count} ASICs found and will power off hash board {board}"))
|
|
||||||
seen_errors.add((log_file, "ASIC Error", f"Chain {chain} has failed with {found_asic_count} ASICs found and will power off hash board {board}"))
|
|
||||||
|
|
||||||
# Check for EEPROM errors
|
|
||||||
for match in eeprom_error_pattern.finditer(log_content):
|
|
||||||
chain = match.group(1)
|
|
||||||
if (log_file, "EEPROM Error", f"Data load fail for chain {chain}") not in seen_errors:
|
|
||||||
results.append((current_date, worker_id, ip, log_file, "EEPROM Error", f"Data load fail for chain {chain}"))
|
|
||||||
seen_errors.add((log_file, "EEPROM Error", f"Data load fail for chain {chain}"))
|
|
||||||
|
|
||||||
# Check for chip bin errors
|
|
||||||
for match in chip_bin_pattern.finditer(log_content):
|
|
||||||
chain = match.group(1)
|
|
||||||
if (log_file, "Chip Bin Error", f"No chip bin for chain {chain}") not in seen_errors:
|
|
||||||
results.append((current_date, worker_id, ip, log_file, "Chip Bin Error", f"No chip bin for chain {chain}"))
|
|
||||||
seen_errors.add((log_file, "Chip Bin Error", f"No chip bin for chain {chain}"))
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error checking logs on {ip}: {e}")
|
logging.error(f"Failed to connect to {ip} with {username}:{password} - {e}")
|
||||||
return logs, asic_errors, results
|
return None
|
||||||
|
|
||||||
|
# Function to execute a command via SSH and return the output
|
||||||
|
def execute_ssh_command(ssh_client, command):
|
||||||
|
try:
|
||||||
|
stdin, stdout, stderr = ssh_client.exec_command(command)
|
||||||
|
return stdout.read().decode('utf-8')
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f"Error executing command '{command}': {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
# Function to get worker ID
|
# Function to get worker ID
|
||||||
def get_worker_id(ssh_client):
|
def get_worker_id(ssh_client):
|
||||||
try:
|
config_content = execute_ssh_command(ssh_client, "cat /config/cgminer.conf")
|
||||||
print("Getting worker ID")
|
if config_content:
|
||||||
stdin, stdout, stderr = ssh_client.exec_command("cat /config/cgminer.conf")
|
|
||||||
config_content = stdout.read().decode('utf-8')
|
|
||||||
# Extract the worker ID from the user field
|
|
||||||
match = re.search(r'"user" *: *"[^.]*\.(\w+)"', config_content)
|
match = re.search(r'"user" *: *"[^.]*\.(\w+)"', config_content)
|
||||||
if match:
|
if match:
|
||||||
worker_id = match.group(1)
|
return match.group(1)
|
||||||
print(f"Got Worker ID: {worker_id}")
|
return "Unknown"
|
||||||
else:
|
|
||||||
worker_id = "Unknown"
|
# Function to check log files for keywords and ASIC errors
|
||||||
except Exception as e:
|
def check_logs(ip, ssh_client, worker_id, current_date, error_keywords):
|
||||||
print(f"Error getting worker ID: {e}")
|
logs = []
|
||||||
worker_id = "Unknown"
|
asic_errors = set() # Using set to avoid duplicate errors
|
||||||
return worker_id
|
results = [] # Using list to avoid duplicate entries
|
||||||
|
log_files_content = execute_ssh_command(ssh_client, "find /var/log/ -type f")
|
||||||
|
if log_files_content:
|
||||||
|
log_files = log_files_content.splitlines()
|
||||||
|
for log_file in log_files:
|
||||||
|
log_content = execute_ssh_command(ssh_client, f"cat {log_file}")
|
||||||
|
if log_content:
|
||||||
|
seen_errors = set()
|
||||||
|
for keyword, error_type in error_keywords.items():
|
||||||
|
if keyword in log_content and (log_file, error_type, keyword) not in seen_errors:
|
||||||
|
logs.append((log_file, error_type, keyword))
|
||||||
|
seen_errors.add((log_file, error_type, keyword))
|
||||||
|
|
||||||
|
for match in asic_pattern.finditer(log_content):
|
||||||
|
chain, asic_count = match.groups()
|
||||||
|
asic_errors.add((chain, int(asic_count)))
|
||||||
|
|
||||||
|
for match in power_off_pattern.finditer(log_content):
|
||||||
|
chain, found_asic_count, board = match.groups()
|
||||||
|
chain = int(chain)
|
||||||
|
found_asic_count = int(found_asic_count)
|
||||||
|
if (log_file, ASIC_ERROR, f"Chain {chain} has failed with {found_asic_count} ASICs found and will power off hash board {board}") not in seen_errors:
|
||||||
|
results.append((current_date, worker_id, ip, log_file, ASIC_ERROR, f"Chain {chain} has failed with {found_asic_count} ASICs found and will power off hash board {board}"))
|
||||||
|
seen_errors.add((log_file, ASIC_ERROR, f"Chain {chain} has failed with {found_asic_count} ASICs found and will power off hash board {board}"))
|
||||||
|
|
||||||
|
for match in eeprom_error_pattern.finditer(log_content):
|
||||||
|
chain = match.group(1)
|
||||||
|
if (log_file, EEPROM_ERROR, f"Data load fail for chain {chain}") not in seen_errors:
|
||||||
|
results.append((current_date, worker_id, ip, log_file, EEPROM_ERROR, f"Data load fail for chain {chain}"))
|
||||||
|
seen_errors.add((log_file, EEPROM_ERROR, f"Data load fail for chain {chain}"))
|
||||||
|
|
||||||
|
for match in chip_bin_pattern.finditer(log_content):
|
||||||
|
chain = match.group(1)
|
||||||
|
if (log_file, CHIP_BIN_ERROR, f"No chip bin for chain {chain}") not in seen_errors:
|
||||||
|
results.append((current_date, worker_id, ip, log_file, CHIP_BIN_ERROR, f"No chip bin for chain {chain}"))
|
||||||
|
seen_errors.add((log_file, CHIP_BIN_ERROR, f"No chip bin for chain {chain}"))
|
||||||
|
return logs, asic_errors, results
|
||||||
|
|
||||||
|
# Function to write results to a text file in the specified format
|
||||||
|
def write_text_file(file_path, results):
|
||||||
|
with open(file_path, 'w') as file:
|
||||||
|
current_worker = None
|
||||||
|
for result in results:
|
||||||
|
date, worker_id, ip, log_file, error_type, error_message = result
|
||||||
|
if worker_id != current_worker:
|
||||||
|
if current_worker is not None:
|
||||||
|
file.write("\n") # Add a blank line between different workers
|
||||||
|
file.write(f"{worker_id}\n")
|
||||||
|
current_worker = worker_id
|
||||||
|
file.write(f"- {error_type}\n")
|
||||||
|
file.write(f"--- {error_message}\n")
|
||||||
|
file.write(f"-" * 80 + "\n")
|
||||||
|
|
||||||
# Main function to iterate over IPs and check for errors
|
# Main function to iterate over IPs and check for errors
|
||||||
def main():
|
def main():
|
||||||
|
@ -111,46 +129,46 @@ def main():
|
||||||
current_date = datetime.now().strftime('%Y-%m-%d')
|
current_date = datetime.now().strftime('%Y-%m-%d')
|
||||||
|
|
||||||
for ip in ips:
|
for ip in ips:
|
||||||
print(f"Processing IP: {ip}")
|
logging.info(f"Processing IP: {ip}")
|
||||||
connected = False
|
connected = False
|
||||||
for os_type, creds in credentials.items():
|
for os_type, creds in credentials.items():
|
||||||
if connected:
|
if connected:
|
||||||
break
|
break
|
||||||
for username, password in creds:
|
for username, password in creds:
|
||||||
ssh_client = paramiko.SSHClient()
|
ssh_client = establish_ssh_connection(ip, username, password)
|
||||||
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
if ssh_client:
|
||||||
try:
|
|
||||||
print(f"Trying {username}:{password} on {ip}")
|
|
||||||
ssh_client.connect(ip, username=username, password=password)
|
|
||||||
connected = True
|
connected = True
|
||||||
worker_id = get_worker_id(ssh_client)
|
worker_id = get_worker_id(ssh_client)
|
||||||
logs, asic_errors, asic_results = check_logs(ip, ssh_client, worker_id, current_date)
|
logs, asic_errors, asic_results = check_logs(ip, ssh_client, worker_id, current_date, error_keywords)
|
||||||
results.extend(asic_results)
|
results.extend(asic_results)
|
||||||
for log in logs:
|
for log in logs:
|
||||||
results.append((current_date, worker_id, ip, log[0], log[1], log[2]))
|
results.append((current_date, worker_id, ip, log[0], log[1], log[2]))
|
||||||
|
|
||||||
unique_asic_errors = {} # Using a dictionary to store chain and failed check count.
|
unique_asic_errors = {} # Using a dictionary to store chain and failed check count.
|
||||||
for chain, asic_count in asic_errors:
|
for chain, asic_count in asic_errors:
|
||||||
failed_checks = unique_asic_errors.get(chain, 0) + 1
|
failed_checks = unique_asic_errors.get(chain, 0) + 1
|
||||||
unique_asic_errors[chain] = failed_checks
|
unique_asic_errors[chain] = failed_checks
|
||||||
if asic_count == 0 and failed_checks == 3:
|
if asic_count == 0 and failed_checks == 3:
|
||||||
results.append((current_date, worker_id, ip, log[0], "ASIC Error", f"Chain {chain} has 3 failed checks with {asic_count} ASICs found"))
|
results.append((current_date, worker_id, ip, "N/A", ASIC_ERROR, f"Chain {chain} has 3 failed checks with {asic_count} ASICs found"))
|
||||||
|
|
||||||
ssh_client.close()
|
ssh_client.close()
|
||||||
break
|
break
|
||||||
except Exception as e:
|
|
||||||
print(f"Connection failed for {ip} with {username}:{password} - {e}")
|
|
||||||
ssh_client.close()
|
|
||||||
|
|
||||||
# Write results to CSV
|
# Write results to CSV
|
||||||
csv_file = 'results.csv'
|
csv_file = 'results.csv'
|
||||||
print(f"Writing results to {csv_file}")
|
logging.info(f"Writing results to {csv_file}")
|
||||||
with open(csv_file, 'w', newline='') as file:
|
with open(csv_file, 'w', newline='') as file:
|
||||||
writer = csv.writer(file)
|
writer = csv.writer(file)
|
||||||
writer.writerow(["Date", "Worker ID", "IP Address", "Log File", "Error Type", "Error Message"])
|
writer.writerow(["Date", "Worker ID", "IP Address", "Log File", "Error Type", "Error Message"])
|
||||||
for result in results:
|
for result in results:
|
||||||
writer.writerow(result)
|
writer.writerow(result)
|
||||||
print("Done")
|
|
||||||
|
# Write results to text file
|
||||||
|
text_file = 'results.txt'
|
||||||
|
logging.info(f"Writing results to {text_file}")
|
||||||
|
write_text_file(text_file, results)
|
||||||
|
|
||||||
|
logging.info("Done")
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
# Load credentials and error keywords
|
# Load credentials and error keywords
|
||||||
|
|
21
ips.txt
21
ips.txt
|
@ -1,20 +1 @@
|
||||||
10.0.90.105
|
192.168.1.171
|
||||||
10.0.80.243
|
|
||||||
10.0.60.194
|
|
||||||
10.0.60.189
|
|
||||||
10.0.50.164
|
|
||||||
10.0.50.28
|
|
||||||
10.0.50.156
|
|
||||||
10.0.40.191
|
|
||||||
10.0.40.118
|
|
||||||
10.0.40.189
|
|
||||||
10.0.40.155
|
|
||||||
10.0.40.244
|
|
||||||
10.0.40.203
|
|
||||||
10.0.30.178
|
|
||||||
10.0.20.163
|
|
||||||
10.0.20.59
|
|
||||||
10.0.20.210
|
|
||||||
10.0.20.131
|
|
||||||
10.0.10.169
|
|
||||||
10.0.100.54
|
|
||||||
|
|
301
qt.py
Normal file
301
qt.py
Normal file
|
@ -0,0 +1,301 @@
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import paramiko
|
||||||
|
import re
|
||||||
|
import json
|
||||||
|
import csv
|
||||||
|
import logging
|
||||||
|
from datetime import datetime
|
||||||
|
from PyQt5.QtWidgets import QApplication, QWidget, QVBoxLayout, QPushButton, QLabel, QFileDialog, QTextEdit, QTreeWidget, QTreeWidgetItem, QSplitter, QHBoxLayout, QPlainTextEdit
|
||||||
|
from PyQt5.QtCore import Qt
|
||||||
|
|
||||||
|
# Constants for error types
|
||||||
|
ASIC_ERROR = "ASIC Error"
|
||||||
|
EEPROM_ERROR = "EEPROM Error"
|
||||||
|
CHIP_BIN_ERROR = "Chip Bin Error"
|
||||||
|
|
||||||
|
# Logging configuration
|
||||||
|
logging.basicConfig(filename='miner_logs.log', level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||||
|
|
||||||
|
# Load credentials from a JSON file
|
||||||
|
def load_credentials(file_path):
|
||||||
|
with open(file_path, 'r') as file:
|
||||||
|
return json.load(file)
|
||||||
|
|
||||||
|
# Load error keywords and types from a JSON file
|
||||||
|
def load_error_keywords(file_path):
|
||||||
|
with open(file_path, 'r') as file:
|
||||||
|
return json.load(file)['error_keywords']
|
||||||
|
|
||||||
|
# Regex patterns for ASIC chip errors and power-off messages
|
||||||
|
asic_pattern = re.compile(r"Chain\[(\d+)\]: find (\d+) asic, times \d+")
|
||||||
|
power_off_pattern = re.compile(r"Chain (\d+) only find (\d+) asic, will power off hash board (\d+)")
|
||||||
|
eeprom_error_pattern = re.compile(r"Data load fail for chain (\d+)\.")
|
||||||
|
chip_bin_pattern = re.compile(r"No chip bin, chain = (\d+)")
|
||||||
|
|
||||||
|
# Function to read IP addresses from a file
|
||||||
|
def read_ips(file_path):
|
||||||
|
with open(file_path, 'r') as file:
|
||||||
|
ips = file.readlines()
|
||||||
|
return [ip.strip() for ip in ips]
|
||||||
|
|
||||||
|
# Function to establish an SSH connection
|
||||||
|
def establish_ssh_connection(ip, username, password):
|
||||||
|
ssh_client = paramiko.SSHClient()
|
||||||
|
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||||
|
try:
|
||||||
|
ssh_client.connect(ip, username=username, password=password, timeout=5)
|
||||||
|
logging.info(f"Connected to {ip} with {username}")
|
||||||
|
return ssh_client
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f"Failed to connect to {ip} with {username}:{password} - {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Function to execute a command via SSH and return the output
|
||||||
|
def execute_ssh_command(ssh_client, command):
|
||||||
|
try:
|
||||||
|
stdin, stdout, stderr = ssh_client.exec_command(command)
|
||||||
|
return stdout.read().decode('utf-8')
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f"Error executing command '{command}': {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Function to get worker ID
|
||||||
|
def get_worker_id(ssh_client):
|
||||||
|
config_content = execute_ssh_command(ssh_client, "cat /config/cgminer.conf")
|
||||||
|
if config_content:
|
||||||
|
match = re.search(r'"user" *: *"[^.]*\.(\w+)"', config_content)
|
||||||
|
if match:
|
||||||
|
return match.group(1)
|
||||||
|
return "Unknown"
|
||||||
|
|
||||||
|
# Function to check log files for keywords and ASIC errors
|
||||||
|
def check_logs(ip, ssh_client, worker_id, current_date, error_keywords):
|
||||||
|
logs = []
|
||||||
|
asic_errors = set() # Using set to avoid duplicate errors
|
||||||
|
results = [] # Using list to avoid duplicate entries
|
||||||
|
log_files_content = execute_ssh_command(ssh_client, "find /var/log/ -type f")
|
||||||
|
if log_files_content:
|
||||||
|
log_files = log_files_content.splitlines()
|
||||||
|
for log_file in log_files:
|
||||||
|
log_content = execute_ssh_command(ssh_client, f"cat {log_file}")
|
||||||
|
if log_content:
|
||||||
|
seen_errors = set()
|
||||||
|
for keyword, error_type in error_keywords.items():
|
||||||
|
if keyword in log_content and (log_file, error_type, keyword) not in seen_errors:
|
||||||
|
logs.append((log_file, error_type, keyword))
|
||||||
|
seen_errors.add((log_file, error_type, keyword))
|
||||||
|
|
||||||
|
for match in asic_pattern.finditer(log_content):
|
||||||
|
chain, asic_count = match.groups()
|
||||||
|
asic_errors.add((chain, int(asic_count)))
|
||||||
|
|
||||||
|
for match in power_off_pattern.finditer(log_content):
|
||||||
|
chain, found_asic_count, board = match.groups()
|
||||||
|
chain = int(chain)
|
||||||
|
found_asic_count = int(found_asic_count)
|
||||||
|
if (log_file, ASIC_ERROR, f"Chain {chain} has failed with {found_asic_count} ASICs found and will power off hash board {board}") not in seen_errors:
|
||||||
|
results.append((current_date, worker_id, ip, log_file, ASIC_ERROR, f"Chain {chain} has failed with {found_asic_count} ASICs found and will power off hash board {board}"))
|
||||||
|
seen_errors.add((log_file, ASIC_ERROR, f"Chain {chain} has failed with {found_asic_count} ASICs found and will power off hash board {board}"))
|
||||||
|
|
||||||
|
for match in eeprom_error_pattern.finditer(log_content):
|
||||||
|
chain = match.group(1)
|
||||||
|
if (log_file, EEPROM_ERROR, f"Data load fail for chain {chain}") not in seen_errors:
|
||||||
|
results.append((current_date, worker_id, ip, log_file, EEPROM_ERROR, f"Data load fail for chain {chain}"))
|
||||||
|
seen_errors.add((log_file, EEPROM_ERROR, f"Data load fail for chain {chain}"))
|
||||||
|
|
||||||
|
for match in chip_bin_pattern.finditer(log_content):
|
||||||
|
chain = match.group(1)
|
||||||
|
if (log_file, CHIP_BIN_ERROR, f"No chip bin for chain {chain}") not in seen_errors:
|
||||||
|
results.append((current_date, worker_id, ip, log_file, CHIP_BIN_ERROR, f"No chip bin for chain {chain}"))
|
||||||
|
seen_errors.add((log_file, CHIP_BIN_ERROR, f"No chip bin for chain {chain}"))
|
||||||
|
return logs, asic_errors, results
|
||||||
|
|
||||||
|
# Function to write results to a text file in the specified format
|
||||||
|
def write_text_file(file_path, results):
|
||||||
|
with open(file_path, 'w') as file:
|
||||||
|
current_worker = None
|
||||||
|
for result in results:
|
||||||
|
date, worker_id, ip, log_file, error_type, error_message = result
|
||||||
|
if worker_id != current_worker:
|
||||||
|
if current_worker is not None:
|
||||||
|
file.write("\n") # Add a blank line between different workers
|
||||||
|
file.write(f"{worker_id}\n")
|
||||||
|
current_worker = worker_id
|
||||||
|
file.write(f"- {error_type}\n")
|
||||||
|
file.write(f"--- {error_message}\n")
|
||||||
|
file.write(f"-" * 80 + "\n")
|
||||||
|
|
||||||
|
# Function to browse for a file
|
||||||
|
def browse_file(label):
|
||||||
|
options = QFileDialog.Options()
|
||||||
|
options |= QFileDialog.ReadOnly
|
||||||
|
file_path, _ = QFileDialog.getOpenFileName(None, "Select File", "", "All Files (*);;Text Files (*.txt)", options=options)
|
||||||
|
if file_path:
|
||||||
|
label.setText(file_path)
|
||||||
|
|
||||||
|
# Main application class
|
||||||
|
class MinerApp(QWidget):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.initUI()
|
||||||
|
|
||||||
|
def initUI(self):
|
||||||
|
main_layout = QVBoxLayout()
|
||||||
|
|
||||||
|
# File path labels
|
||||||
|
self.ips_label = QLabel("No IPs file selected")
|
||||||
|
self.credentials_label = QLabel("No credentials file selected")
|
||||||
|
self.errors_label = QLabel("No errors file selected")
|
||||||
|
|
||||||
|
# Set default paths if files exist
|
||||||
|
self.set_default_paths()
|
||||||
|
|
||||||
|
# Text edit for logs
|
||||||
|
self.logs_text = QPlainTextEdit()
|
||||||
|
self.logs_text.setReadOnly(True)
|
||||||
|
|
||||||
|
# Tree views for machines and errors
|
||||||
|
self.machines_tree = QTreeWidget()
|
||||||
|
self.machines_tree.setHeaderLabel("Machines")
|
||||||
|
self.machines_tree.itemClicked.connect(self.display_errors)
|
||||||
|
|
||||||
|
self.errors_tree = QTreeWidget()
|
||||||
|
self.errors_tree.setHeaderLabel("Errors")
|
||||||
|
|
||||||
|
# Buttons
|
||||||
|
browse_ips_btn = QPushButton('Browse IPs File')
|
||||||
|
browse_ips_btn.clicked.connect(lambda: browse_file(self.ips_label))
|
||||||
|
|
||||||
|
browse_credentials_btn = QPushButton('Browse Credentials File')
|
||||||
|
browse_credentials_btn.clicked.connect(lambda: browse_file(self.credentials_label))
|
||||||
|
|
||||||
|
browse_errors_btn = QPushButton('Browse Errors File')
|
||||||
|
browse_errors_btn.clicked.connect(lambda: browse_file(self.errors_label))
|
||||||
|
|
||||||
|
start_btn = QPushButton('Start')
|
||||||
|
start_btn.clicked.connect(self.start_process)
|
||||||
|
|
||||||
|
# Splitter for tree views
|
||||||
|
tree_splitter = QSplitter()
|
||||||
|
tree_splitter.addWidget(self.machines_tree)
|
||||||
|
tree_splitter.addWidget(self.errors_tree)
|
||||||
|
|
||||||
|
# Splitter for the main layout and logs
|
||||||
|
main_splitter = QSplitter(Qt.Vertical)
|
||||||
|
main_splitter.addWidget(tree_splitter)
|
||||||
|
main_splitter.addWidget(self.logs_text)
|
||||||
|
main_splitter.setSizes([500, 100]) # Initial sizes: larger for the tree views, smaller for the logs
|
||||||
|
|
||||||
|
# Add widgets to layout
|
||||||
|
main_layout.addWidget(browse_ips_btn)
|
||||||
|
main_layout.addWidget(self.ips_label)
|
||||||
|
main_layout.addWidget(browse_credentials_btn)
|
||||||
|
main_layout.addWidget(self.credentials_label)
|
||||||
|
main_layout.addWidget(browse_errors_btn)
|
||||||
|
main_layout.addWidget(self.errors_label)
|
||||||
|
main_layout.addWidget(start_btn)
|
||||||
|
main_layout.addWidget(main_splitter)
|
||||||
|
|
||||||
|
self.setLayout(main_layout)
|
||||||
|
self.setWindowTitle('Miner Error Checker')
|
||||||
|
self.show()
|
||||||
|
|
||||||
|
def set_default_paths(self):
|
||||||
|
default_ips_path = "ips.txt"
|
||||||
|
default_credentials_path = "credentials.json"
|
||||||
|
default_errors_path = "errors.json"
|
||||||
|
|
||||||
|
if os.path.exists(default_ips_path):
|
||||||
|
self.ips_label.setText(default_ips_path)
|
||||||
|
if os.path.exists(default_credentials_path):
|
||||||
|
self.credentials_label.setText(default_credentials_path)
|
||||||
|
if os.path.exists(default_errors_path):
|
||||||
|
self.errors_label.setText(default_errors_path)
|
||||||
|
|
||||||
|
def start_process(self):
|
||||||
|
ips_path = self.ips_label.text()
|
||||||
|
credentials_path = self.credentials_label.text()
|
||||||
|
errors_path = self.errors_label.text()
|
||||||
|
|
||||||
|
if ips_path == "No IPs file selected" or credentials_path == "No credentials file selected" or errors_path == "No errors file selected":
|
||||||
|
self.logs_text.appendPlainText("Please select all required files.")
|
||||||
|
return
|
||||||
|
|
||||||
|
credentials = load_credentials(credentials_path)
|
||||||
|
error_keywords = load_error_keywords(errors_path)
|
||||||
|
ips = read_ips(ips_path)
|
||||||
|
|
||||||
|
results = []
|
||||||
|
current_date = datetime.now().strftime('%Y-%m-%d')
|
||||||
|
|
||||||
|
self.machines_tree.clear()
|
||||||
|
self.errors_tree.clear()
|
||||||
|
|
||||||
|
for ip in ips:
|
||||||
|
self.logs_text.appendPlainText(f"Processing IP: {ip}")
|
||||||
|
logging.info(f"Processing IP: {ip}")
|
||||||
|
connected = False
|
||||||
|
for os_type, creds in credentials.items():
|
||||||
|
if connected:
|
||||||
|
break
|
||||||
|
for username, password in creds:
|
||||||
|
ssh_client = establish_ssh_connection(ip, username, password)
|
||||||
|
if ssh_client:
|
||||||
|
connected = True
|
||||||
|
worker_id = get_worker_id(ssh_client)
|
||||||
|
logs, asic_errors, asic_results = check_logs(ip, ssh_client, worker_id, current_date, error_keywords)
|
||||||
|
results.extend(asic_results)
|
||||||
|
for log in logs:
|
||||||
|
results.append((current_date, worker_id, ip, log[0], log[1], log[2]))
|
||||||
|
|
||||||
|
unique_asic_errors = {}
|
||||||
|
for chain, asic_count in asic_errors:
|
||||||
|
failed_checks = unique_asic_errors.get(chain, 0) + 1
|
||||||
|
unique_asic_errors[chain] = failed_checks
|
||||||
|
if asic_count == 0 and failed_checks == 3:
|
||||||
|
results.append((current_date, worker_id, ip, "N/A", ASIC_ERROR, f"Chain {chain} has 3 failed checks with {asic_count} ASICs found"))
|
||||||
|
|
||||||
|
self.add_machine_item(ip, worker_id, results)
|
||||||
|
ssh_client.close()
|
||||||
|
break
|
||||||
|
|
||||||
|
# Write results to CSV
|
||||||
|
csv_file = 'results.csv'
|
||||||
|
logging.info(f"Writing results to {csv_file}")
|
||||||
|
with open(csv_file, 'w', newline='') as file:
|
||||||
|
writer = csv.writer(file)
|
||||||
|
writer.writerow(["Date", "Worker ID", "IP Address", "Log File", "Error Type", "Error Message"])
|
||||||
|
for result in results:
|
||||||
|
writer.writerow(result)
|
||||||
|
|
||||||
|
# Write results to text file
|
||||||
|
text_file = 'results.txt'
|
||||||
|
logging.info(f"Writing results to {text_file}")
|
||||||
|
write_text_file(text_file, results)
|
||||||
|
|
||||||
|
self.logs_text.appendPlainText("Process completed. Results saved to results.csv and results.txt.")
|
||||||
|
|
||||||
|
def add_machine_item(self, ip, worker_id, results):
|
||||||
|
machine_item = QTreeWidgetItem([f"{worker_id} ({ip})"])
|
||||||
|
machine_item.setData(0, 1, results) # Store results in the item for later retrieval
|
||||||
|
self.machines_tree.addTopLevelItem(machine_item)
|
||||||
|
|
||||||
|
def display_errors(self, item, column):
|
||||||
|
self.errors_tree.clear()
|
||||||
|
results = item.data(0, 1) # Retrieve stored results from the item
|
||||||
|
if results:
|
||||||
|
for result in results:
|
||||||
|
date, worker_id, ip, log_file, error_type, error_message = result
|
||||||
|
error_item = QTreeWidgetItem([f"{error_type}: {error_message}"])
|
||||||
|
self.errors_tree.addTopLevelItem(error_item)
|
||||||
|
|
||||||
|
# Main function to run the PyQt5 application
|
||||||
|
def main():
|
||||||
|
app = QApplication(sys.argv)
|
||||||
|
ex = MinerApp()
|
||||||
|
sys.exit(app.exec_())
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
# lol
|
66
results.csv
66
results.csv
|
@ -1,66 +0,0 @@
|
||||||
Date,Worker ID,IP Address,Log File,Error Type,Error Message
|
|
||||||
2024-05-28,mw9875,10.0.90.105,/var/log/miner.log,EEPROM Error,Data load fail for chain 1
|
|
||||||
2024-05-28,mw9875,10.0.90.105,/var/log/miner.log,EEPROM Error,Data load fail for chain 2
|
|
||||||
2024-05-28,mw9875,10.0.90.105,/var/log/messages,EEPROM Error,Data load fail for chain 1
|
|
||||||
2024-05-28,mw9875,10.0.90.105,/var/log/messages,EEPROM Error,Data load fail for chain 2
|
|
||||||
2024-05-28,mw8576,10.0.80.243,/var/log/miner.log,black hole,reg crc error
|
|
||||||
2024-05-28,mw6643,10.0.60.194,/var/log/miner.log,ASIC Error,Chain 0 has failed with 0 ASICs found and will power off hash board 0
|
|
||||||
2024-05-28,mw6643,10.0.60.194,/var/log/miner/miner.log,ASIC Error,Chain 0 has failed with 0 ASICs found and will power off hash board 0
|
|
||||||
2024-05-28,mw6643,10.0.60.194,/var/log/miner/miner.log.0,ASIC Error,Chain 0 has failed with 0 ASICs found and will power off hash board 0
|
|
||||||
2024-05-28,mw6643,10.0.60.194,/var/log/miner/miner.log.1,ASIC Error,Chain 0 has failed with 0 ASICs found and will power off hash board 0
|
|
||||||
2024-05-28,mw6643,10.0.60.194,/var/log/messages,ASIC Error,Chain 0 has failed with 0 ASICs found and will power off hash board 0
|
|
||||||
2024-05-28,mw6643,10.0.60.194,/var/log/miner.log,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw6643,10.0.60.194,/var/log/miner/miner.log,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw6643,10.0.60.194,/var/log/miner/miner.log.0,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw6643,10.0.60.194,/var/log/miner/miner.log.1,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw6643,10.0.60.194,/var/log/messages,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw5513,10.0.50.164,/var/log/miner.log,ASIC Error,Chain 1 has failed with 0 ASICs found and will power off hash board 1
|
|
||||||
2024-05-28,mw5513,10.0.50.164,/var/log/messages,ASIC Error,Chain 1 has failed with 0 ASICs found and will power off hash board 1
|
|
||||||
2024-05-28,mw5513,10.0.50.164,/var/log/miner.log,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw5513,10.0.50.164,/var/log/messages,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw5423,10.0.50.28,/var/log/miner.log,ASIC Error,Chain 0 has failed with 0 ASICs found and will power off hash board 0
|
|
||||||
2024-05-28,mw5423,10.0.50.28,/var/log/messages,ASIC Error,Chain 0 has failed with 0 ASICs found and will power off hash board 0
|
|
||||||
2024-05-28,mw5383,10.0.50.156,/var/log/miner.log,black hole,reg crc error
|
|
||||||
2024-05-28,mw5383,10.0.50.156,/var/log/messages,black hole,reg crc error
|
|
||||||
2024-05-28,mw4184,10.0.40.244,/var/log/miner.log,ASIC Error,Chain 0 has failed with 96 ASICs found and will power off hash board 0
|
|
||||||
2024-05-28,mw4184,10.0.40.244,/var/log/messages,ASIC Error,Chain 0 has failed with 96 ASICs found and will power off hash board 0
|
|
||||||
2024-05-28,mw4184,10.0.40.244,/var/log/miner.log,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw4184,10.0.40.244,/var/log/messages,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw4122,10.0.40.203,/var/log/miner.log,ASIC Error,Chain 2 has failed with 0 ASICs found and will power off hash board 2
|
|
||||||
2024-05-28,mw4122,10.0.40.203,/var/log/messages,ASIC Error,Chain 2 has failed with 0 ASICs found and will power off hash board 2
|
|
||||||
2024-05-28,mw4122,10.0.40.203,/var/log/miner.log,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw4122,10.0.40.203,/var/log/messages,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw3514,10.0.30.178,/var/log/miner.log,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw3514,10.0.30.178,/var/log/miner.log,PSU,bitmain_get_power_status failed
|
|
||||||
2024-05-28,mw3514,10.0.30.178,/var/log/miner.log,PSU,power voltage can not meet the target
|
|
||||||
2024-05-28,mw3514,10.0.30.178,/var/log/messages,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw3514,10.0.30.178,/var/log/messages,PSU,bitmain_get_power_status failed
|
|
||||||
2024-05-28,mw3514,10.0.30.178,/var/log/messages,PSU,power voltage can not meet the target
|
|
||||||
2024-05-28,mw2564,10.0.20.163,/var/log/messages,ASIC Error,Chain 0 has failed with 0 ASICs found and will power off hash board 0
|
|
||||||
2024-05-28,mw2564,10.0.20.163,/var/log/messages,ASIC Error,Chain 1 has failed with 0 ASICs found and will power off hash board 1
|
|
||||||
2024-05-28,mw2564,10.0.20.163,/var/log/messages,ASIC Error,Chain 2 has failed with 0 ASICs found and will power off hash board 2
|
|
||||||
2024-05-28,mw2564,10.0.20.163,/var/log/messages,EEPROM Error,Data load fail for chain 1
|
|
||||||
2024-05-28,mw2564,10.0.20.163,/var/log/messages,Chip Bin Error,No chip bin for chain 0
|
|
||||||
2024-05-28,mw2564,10.0.20.163,/var/log/messages,Chip Bin Error,No chip bin for chain 1
|
|
||||||
2024-05-28,mw2564,10.0.20.163,/var/log/messages,Chip Bin Error,No chip bin for chain 2
|
|
||||||
2024-05-28,mw2564,10.0.20.163,/var/log/messages,Temperature Error,ERROR_TEMP_TOO_HIGH
|
|
||||||
2024-05-28,mw2564,10.0.20.163,/var/log/messages,PIC Error,_pic_write_iic failed!
|
|
||||||
2024-05-28,mw2564,10.0.20.163,/var/log/messages,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw2481,10.0.20.59,/var/log/messages,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw2184,10.0.20.210,/var/log/messages,ASIC Error,Chain 0 has failed with 0 ASICs found and will power off hash board 0
|
|
||||||
2024-05-28,mw2144,10.0.20.131,/var/log/messages,ASIC Error,Chain 0 has failed with 0 ASICs found and will power off hash board 0
|
|
||||||
2024-05-28,mw2144,10.0.20.131,/var/log/messages,ASIC Error,Chain 1 has failed with 0 ASICs found and will power off hash board 1
|
|
||||||
2024-05-28,mw2144,10.0.20.131,/var/log/messages,ASIC Error,Chain 2 has failed with 0 ASICs found and will power off hash board 2
|
|
||||||
2024-05-28,mw2144,10.0.20.131,/var/log/messages,EEPROM Error,Data load fail for chain 0
|
|
||||||
2024-05-28,mw2144,10.0.20.131,/var/log/messages,Chip Bin Error,No chip bin for chain 0
|
|
||||||
2024-05-28,mw2144,10.0.20.131,/var/log/messages,Chip Bin Error,No chip bin for chain 1
|
|
||||||
2024-05-28,mw2144,10.0.20.131,/var/log/messages,Chip Bin Error,No chip bin for chain 2
|
|
||||||
2024-05-28,mw2144,10.0.20.131,/var/log/messages,Temperature Error,ERROR_TEMP_TOO_HIGH
|
|
||||||
2024-05-28,mw2144,10.0.20.131,/var/log/messages,PIC Error,_pic_write_iic failed!
|
|
||||||
2024-05-28,mw2144,10.0.20.131,/var/log/messages,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw1132,10.0.10.169,/var/log/miner.log,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mw1132,10.0.10.169,/var/log/messages,SoC failure,ERROR_SOC_INIT: soc init failed
|
|
||||||
2024-05-28,mwxxx2,10.0.100.54,/var/log/miner.log,ASIC Error,Chain 1 has failed with 0 ASICs found and will power off hash board 1
|
|
||||||
2024-05-28,mwxxx2,10.0.100.54,/var/log/miner.log,ASIC Error,Chain 2 has failed with 0 ASICs found and will power off hash board 2
|
|
||||||
2024-05-28,mwxxx2,10.0.100.54,/var/log/messages,ASIC Error,Chain 1 has failed with 0 ASICs found and will power off hash board 1
|
|
||||||
2024-05-28,mwxxx2,10.0.100.54,/var/log/messages,ASIC Error,Chain 2 has failed with 0 ASICs found and will power off hash board 2
|
|
|
Loading…
Reference in a new issue