This gets cleaner.

This commit is contained in:
Tristan Smith 2024-05-25 16:38:38 -04:00
parent 6c4bf10bcb
commit e91bb9c89b
5 changed files with 67 additions and 59 deletions

6
credentials.json Normal file
View file

@ -0,0 +1,6 @@
{
"stock": [["miner", "miner"], ["root", "root"]],
"braiins": [["root", "root"]],
"luxos": [["root", "root"]],
// "vnish": [["user", "password"]] // apparently vnish's ssh port is closed? I messaged them on telegram asking about it.
}

14
errors.json Normal file
View file

@ -0,0 +1,14 @@
{
"error_keywords": [
"ERROR_TEMP_TOO_HIGH",
"ERROR_HASHRATE_TOO_LOW",
"ERROR_NETWORK_DISCONNECTED",
"ERROR_POWER_LOST: power voltage rise or drop",
"SWEEP_STRING",
"_pic_write_iic failed!",
"PLL read exceeded wait time",
"ERROR_SOC_INIT: soc init failed",
"fail to read 0:1",
"fail to write 0:1"
]
}

View file

@ -1,31 +1,29 @@
import paramiko
import csv
import re
import json
# Define credentials for different OS types
credentials = {
"stock": [("miner", "miner"), ("root", "root")],
"braiins": [("root", "root")],
"luxos": [("root", "root")],
"vnish": [("user", "password")] # Replace with correct credentials
}
# Load credentials from a JSON file
def load_credentials(file_path):
with open(file_path, 'r') as file:
return json.load(file)
# List of error keywords
error_keywords = [
"ERROR_TEMP_TOO_HIGH",
"ERROR_HASHRATE_TOO_LOW",
"ERROR_NETWORK_DISCONNECTED",
"ERROR_POWER_LOST: power voltage rise or drop",
"SWEEP_STRING",
"_pic_write_iic failed!",
"PLL read exceeded wait time",
"ERROR_SOC_INIT: soc init failed",
"fail to read 0:1",
"fail to write 0:1"
]
# Load error keywords from a JSON file
def load_error_keywords(file_path):
with open(file_path, 'r') as file:
return json.load(file)['error_keywords']
# Regex pattern for ASIC chip errors
# Define paths to the configuration files
CREDENTIALS_FILE = 'credentials.json'
ERRORS_FILE = 'errors.json'
# Load credentials and error keywords
credentials = load_credentials(CREDENTIALS_FILE)
error_keywords = load_error_keywords(ERRORS_FILE)
# Regex patterns for ASIC chip errors and power-off messages
asic_pattern = re.compile(r"Chain\[(\d+)\]: find (\d+) asic, times \d+")
power_off_pattern = re.compile(r"Chain (\d+) only find (\d+) asic, will power off hash board (\d+)")
# Function to read IP addresses from a file
def read_ips(file_path):
@ -34,16 +32,17 @@ def read_ips(file_path):
return [ip.strip() for ip in ips]
# Function to check log files for keywords and ASIC errors
def check_logs(ip, ssh_client):
def check_logs(ip, ssh_client, worker_id):
logs = []
asic_errors = {}
correct_asic_count = {}
asic_errors = set() # Using set to avoid duplicate errors
results = []
try:
print(f"Checking logs on {ip}")
stdin, stdout, stderr = ssh_client.exec_command("find /var/log/ -type f")
log_files = stdout.readlines()
for log_file in log_files:
log_file = log_file.strip()
print(f"Checking file: {log_file}") # Debug statement
# Check if file should be ignored
if log_file.endswith(('tmp', 'utmp', 'btmp', 'wtmp')):
continue
@ -56,24 +55,29 @@ def check_logs(ip, ssh_client):
stdin, stdout, stderr = ssh_client.exec_command(f"cat {log_file}")
log_content = stdout.read().decode('utf-8', errors='ignore')
print(f"Content of {log_file}: {log_content[:500]}") # Debug statement to show part of the log content
for keyword in error_keywords:
if keyword in log_content:
logs.append((log_file, keyword))
# Check for ASIC chip errors
# Check for ASIC chip errors and power-off messages
for match in asic_pattern.finditer(log_content):
chain, asic_count = match.groups()
asic_count = int(asic_count)
if chain not in asic_errors:
asic_errors[chain] = []
asic_errors[chain].append(asic_count)
asic_errors.add((chain, asic_count))
print(f"Chain {chain} has {asic_count} chips.") # Debug statement
# Determine the correct ASIC count for each chain
if chain not in correct_asic_count and asic_count > 0:
correct_asic_count[chain] = asic_count
# Check for power-off messages
for match in power_off_pattern.finditer(log_content):
chain, found_asic_count, board = match.groups()
found_asic_count = int(found_asic_count)
chain = int(chain)
print(f"Power-off message found: Chain {chain}, ASIC count: {found_asic_count}, Board: {board}") # Debug statement
results.append([worker_id, ip, "ASIC Error", f"Chain {chain} has failed with {found_asic_count} ASICs found and will power off hash board {board}"])
except Exception as e:
print(f"Error checking logs on {ip}: {e}")
return logs, asic_errors, correct_asic_count
return logs, asic_errors, results
# Function to get worker ID
def get_worker_id(ssh_client):
@ -85,7 +89,6 @@ def get_worker_id(ssh_client):
match = re.search(r'"user" *: *"[^.]*\.(\w+)"', config_content)
if match:
worker_id = match.group(1)
print("Got worker ID: ", worker_id)
else:
worker_id = "Unknown"
except Exception as e:
@ -112,19 +115,18 @@ def main():
ssh_client.connect(ip, username=username, password=password)
connected = True
worker_id = get_worker_id(ssh_client)
logs, asic_errors, correct_asic_count = check_logs(ip, ssh_client)
logs, asic_errors, asic_results = check_logs(ip, ssh_client, worker_id)
results.extend(asic_results)
for log in logs:
results.append([worker_id, ip, log[0], log[1]])
# Check for ASIC chip errors and add to results
# Maybe think about this differntly. Look for "will power off hashboard" and *then* figure out why the hashboard has been powered off.
for chain, counts in asic_errors.items():
if counts.count(0) >= 3:
results.append([worker_id, ip, "ASIC Error", f"Chain {chain} has {counts.count(0)} failed checks with 0 ASICs found"])
elif chain in correct_asic_count:
expected_count = correct_asic_count[chain]
if any(count != expected_count and count > 0 for count in counts):
results.append([worker_id, ip, "ASIC Discrepancy", f"Chain {chain} has varying ASIC counts: {counts}"])
unique_asic_errors = {} # Using a dictionary to store chain and failed check count.
for chain, asic_count in asic_errors:
for chain, asic_count in asic_errors:
failed_checks = unique_asic_errors.get(chain, 0) + 1 # array
unique_asic_errors[chain] = failed_checks
if asic_count == 0 and failed_checks == 3:
results.append([worker_id, ip, "ASIC Error", f"Chain {chain} has 3 failed checks with {asic_count} ASICs found"])
ssh_client.close()
break
@ -138,6 +140,7 @@ def main():
writer = csv.writer(file)
writer.writerow(["Worker ID", "IP Address", "Log File", "Error"])
writer.writerows(results)
print("Writing: ", results)
print("Done")
if __name__ == "__main__":

View file

@ -1,3 +1,3 @@
# Antminer error finder
Checks for specific strings in log files of Antminer machines. Currently works with stock firmware. Expand to Braiins & LuxOS.
Checks for specific strings in log files of Antminer machines. Currently works with stock firmware (and barely at that). Expand to Braiins & LuxOS.

View file

@ -1,15 +0,0 @@
Worker ID,IP Address,Log File,Error
mw1012,192.168.1.171,/var/log/2024-03/06/cglog_init_2024-03-06_19-00-57/miner.log,ERROR_SOC_INIT: soc init failed
mw1012,192.168.1.171,/var/log/2024-03/06/cglog_init_2024-03-06_19-00-57/messages,ERROR_SOC_INIT: soc init failed
mw1012,192.168.1.171,/var/log/2024-03/06/cglog_init_2024-03-06_18-53-25/miner.log,ERROR_SOC_INIT: soc init failed
mw1012,192.168.1.171,/var/log/2024-03/06/cglog_init_2024-03-06_18-53-25/messages,ERROR_SOC_INIT: soc init failed
mw1012,192.168.1.171,/var/log/2024-03/06/cglog_init_2024-03-06_19-07-47/miner.log,ERROR_SOC_INIT: soc init failed
mw1012,192.168.1.171,/var/log/2024-03/06/cglog_init_2024-03-06_19-07-47/messages,ERROR_SOC_INIT: soc init failed
mw1012,192.168.1.171,/var/log/2024-03/06/cglog_init_2024-03-06_20-51-43/miner.log,ERROR_SOC_INIT: soc init failed
mw1012,192.168.1.171,/var/log/2024-03/06/cglog_init_2024-03-06_20-51-43/messages,ERROR_SOC_INIT: soc init failed
mw1012,192.168.1.171,/var/log/2024-03/06/cglog_init_2024-03-06_18-01-31/miner.log,ERROR_SOC_INIT: soc init failed
mw1012,192.168.1.171,/var/log/2024-03/06/cglog_init_2024-03-06_18-01-31/messages,ERROR_SOC_INIT: soc init failed
mw1012,192.168.1.171,/var/log/test,ERROR_POWER_LOST: power voltage rise or drop
mw1012,192.168.1.171,ASIC Error,Chain 0 has 3 failed checks with 0 ASICs found
mw1012,192.168.1.171,ASIC Discrepancy,"Chain 1 has varying ASIC counts: [110, 110, 110, 110, 110, 110, 104, 104, 104]"
mw1012,192.168.1.171,ASIC Discrepancy,"Chain 2 has varying ASIC counts: [110, 110, 110, 110, 110, 110, 114]"
1 Worker ID IP Address Log File Error
2 mw1012 192.168.1.171 /var/log/2024-03/06/cglog_init_2024-03-06_19-00-57/miner.log ERROR_SOC_INIT: soc init failed
3 mw1012 192.168.1.171 /var/log/2024-03/06/cglog_init_2024-03-06_19-00-57/messages ERROR_SOC_INIT: soc init failed
4 mw1012 192.168.1.171 /var/log/2024-03/06/cglog_init_2024-03-06_18-53-25/miner.log ERROR_SOC_INIT: soc init failed
5 mw1012 192.168.1.171 /var/log/2024-03/06/cglog_init_2024-03-06_18-53-25/messages ERROR_SOC_INIT: soc init failed
6 mw1012 192.168.1.171 /var/log/2024-03/06/cglog_init_2024-03-06_19-07-47/miner.log ERROR_SOC_INIT: soc init failed
7 mw1012 192.168.1.171 /var/log/2024-03/06/cglog_init_2024-03-06_19-07-47/messages ERROR_SOC_INIT: soc init failed
8 mw1012 192.168.1.171 /var/log/2024-03/06/cglog_init_2024-03-06_20-51-43/miner.log ERROR_SOC_INIT: soc init failed
9 mw1012 192.168.1.171 /var/log/2024-03/06/cglog_init_2024-03-06_20-51-43/messages ERROR_SOC_INIT: soc init failed
10 mw1012 192.168.1.171 /var/log/2024-03/06/cglog_init_2024-03-06_18-01-31/miner.log ERROR_SOC_INIT: soc init failed
11 mw1012 192.168.1.171 /var/log/2024-03/06/cglog_init_2024-03-06_18-01-31/messages ERROR_SOC_INIT: soc init failed
12 mw1012 192.168.1.171 /var/log/test ERROR_POWER_LOST: power voltage rise or drop
13 mw1012 192.168.1.171 ASIC Error Chain 0 has 3 failed checks with 0 ASICs found
14 mw1012 192.168.1.171 ASIC Discrepancy Chain 1 has varying ASIC counts: [110, 110, 110, 110, 110, 110, 104, 104, 104]
15 mw1012 192.168.1.171 ASIC Discrepancy Chain 2 has varying ASIC counts: [110, 110, 110, 110, 110, 110, 114]