#!/usr/bin/env python3
import math
import time, logging, os, threading, subprocess, struct, base64, socket, sys, multiprocessing, re, glob, shutil
from msgs.monitor_pb2 import SystemMonitor, SimpleMonitor
from msgs.stub_pb2 import Stubs
import pycrmw, psutil
import sys, time, yaml
import socket, struct, select, threading, time
import traceback
import functools
ERROR_REPORTED = set()
def report_stub(func):
  @functools.wraps(func)
  def wrapper(*args, **kwargs):
    try: return func(*args, **kwargs)
    except Exception as e:
      tb = traceback.extract_tb(e.__traceback__)
      lineno = next((i.lineno for i in reversed(tb) if i.name == func.__name__), 'error')
      err_info = f" {func.__name__}:{lineno} --> {type(e).__name__}: {e}"
      err_key = f" {func.__name__}:{lineno}"
      if err_key not in ERROR_REPORTED:
        ERROR_REPORTED.add(err_key)
        pycrmw.log_error(err_info)
        pycrmw.Stub('MonitorCrash', err_info)
      return None
  return wrapper


def custom_run(command):
  max_retries = 3
  for attempt in range(max_retries):
    try:
      result = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True).communicate()[0].decode('utf-8', "ignore").strip()
      if result:
        return result
    except Exception as e:
      pycrmw.log_error(f"command: '{command}' run {attempt + 1} failed with error: {e}")
  return ""

class Pmbus6(object): # ADU6.0 采用的BMC
  def __parse(self, data, format):
    output = int(data[3:] + data[:2], 16)
    if format == 'vout': return output * 2**self.vout_exp
    if output & 0x8000: exponent = -((output >> 11 & 0xF ^ 15) + 1)
    else: exponent = output >> 11 & 0xF
    if output & 0x400: mantissa = -((output & 0x3FF ^ 1023) + 1)
    else: mantissa = output & 0x7FF
    return mantissa * 2**exponent

  def __init__(self):
    self.name = ''
    ipmi_cmd = 'ipmitool -I lanplus -H 192.168.1.254 -U cowa -P Cowa2022 raw 0x3c 0x62 0x54'  # 远程使用
    if 'adu' in socket.gethostname(): ipmi_cmd = 'ipmitool raw 0x3c 0x62 0x54' # 本机使用
    self.commands = {'vin': ipmi_cmd + ' 0x88 2', 'vout': ipmi_cmd + ' 0x8b 2', 'iout': ipmi_cmd + ' 0x8c 2', 'temp': ipmi_cmd + ' 0x8d 2'}
    self.name = 'huawei'
    self.vout_mode = ipmi_cmd + ' 0x20 1'
    self.vout_exp = -1
    
  def Status(self):
    if self.vout_exp < 0:
      mode = custom_run(self.vout_mode)
      try: self.vout_exp = -((int(mode, 16) & 0xF ^ 15) + 1)
      except: return None
    try:
      results = {'name': self.name}
      for k, v in self.commands.items():
        results[k] = self.__parse(custom_run(v), k)
        if results[k] == None: return None
      results['power'] = results['vout'] * results['iout']
      return results
    except: return None

class Pmbus(object): # ADU5.6 采用的232转PMBUS
  def __comm(self, cmd):
    self.com.flushInput()
    self.com.write(cmd)
    time.sleep(0.02)
    data = self.com.read(self.com.inWaiting())
    return data

  def __parse(self, data, format):
    if data:
      output = struct.unpack('<H', data[:2])[0]
      if format == 'vout': return output * 2**self.vout_exp
      if output & 0x8000: exponent = -((output >> 11 & 0xF ^ 15) + 1)
      else: exponent = output >> 11 & 0xF
      if output & 0x400: mantissa = -((output & 0x3FF ^ 1023) + 1)
      else: mantissa = output & 0x7FF
      return mantissa * 2**exponent
    else: return 0

  def __init__(self):
    self.name = []
    self.num = 0
    import serial
    try: 
      self.com = serial.Serial("/dev/ttyS0", 9600)
    except:
      pycrmw.log_error('/dev/ttyS0 not found')
      return

    i2c_addrs = {'try': [b'\x00'], 'flex': [b'\xff'], 'huawei': [b'\xa9', b'\xad'], 'delta': [b'\xc1']}
    for k, v in i2c_addrs.items():
      vout_mode = self.__comm(v[0] + struct.pack("BB", 0x20, 0x01))
      if vout_mode:
        self.vout_exp = -((vout_mode[0] & 0xF ^ 15) + 1)
        self.name.append(k)
        if len(v) == 2 and self.__comm(v[1] + struct.pack("BB", 0x20, 0x01)): self.name.append(k)
        time.sleep(0.5)
        pycrmw.log_info('power name:%s' % self.name)
        self.num = len(self.name)
        self.commands = [{'vin': i + b'\x88\x02', 'vout': i + b'\x8b\x02', 'iout': i + b'\x8c\x02', 'temp': i + b'\x8d\x02'} for i in v]
        f = '/opt/cowa/.cache/.power_module'
        txt = ','.join(self.name)
        if (not os.path.exists(f)) or open(f, 'r').read() != txt: open(f, 'w').write(txt)
        return
      time.sleep(1)

  def Status(self):
    if len(self.name) == 0: return None
    results = [{} for _ in range(self.num)]
    try: 
      for i in range(self.num):
        for k, v in self.commands[i].items(): results[i][k] = self.__parse(self.__comm(v), k)
        results[i]['power'] = results[i]['vout'] * results[i]['iout']
        results[i]['name'] = self.name[i]
      return results
    except: return None

class Monitor:
  @staticmethod
  def NvidiaGpu():
    r = custom_run('nvidia-smi --query-gpu=temperature.gpu,memory.total,memory.free,utilization.gpu --format=csv,noheader')
    gpus = []
    try:
      for line in r.split('\n'): gpus.append([float(i.split()[0]) for i in line.split(',')])
      return gpus
    except:
      raise Exception(f'GPUFall: {r}')
  
  @staticmethod
  def GetGPUProcess():
    # TODO:
    r = custom_run('nvidia-smi --query-gpu=index,gpu_uuid --format=csv,noheader,nounits')
    s = custom_run('nvidia-smi --query-compute-apps=gpu_uuid,pid,used_gpu_memory --format=csv,noheader,nounits')

    gpu_uuid_dict = {
        # GPU的UUID：GPU编号
        # GPU-UUID: 0 / 1
        gpu_uuid.strip(): int(gpu_num)
        for gpu_num, gpu_uuid in
        (line.split(',') for line in r.split('\n') if line and 'fail' not in line)
    }

    process_gpu_dict = {
        # 进程PID: [进程占用显存, 进程显存占用所属显卡编号]
        # 3501: [1024.00, 80, 0]
        int(pid): [
            float(gpu_mem),
            gpu_uuid_dict[gpu_uuid.strip()]
        ]
        for gpu_uuid, pid, gpu_mem in
        (line.split(',') for line in s.split('\n') if line and 'fail' not in line)
    }

    return process_gpu_dict

  def __init__(self):
    self.ipmi_sensors = []
    self.tick = 0
    self.info = SystemMonitor()
    self.info.sequence = 1
    self.info.cpu.total = multiprocessing.cpu_count()
    self.info.name = socket.gethostname()
    self.get_cpu_temp = None
    if 'adu' in self.info.name: 
      self.adu_power = Pmbus() if 'adu5' in self.info.name else Pmbus6()
      self.get_gpu = Monitor.NvidiaGpu
      self.gpu_error_report = False
    else: self.get_gpu = lambda: []
    self.processes = []
    self.update()

  @report_stub
  def update(self):
    self.info.sequence = self.info.sequence + 1
    self.info.timestamp = int(time.time() * 1e9)
    self.tick += 1
    self.info.cpu.free = (1.0 - psutil.cpu_percent() / 100.0) * self.info.cpu.total
    ps = []
    self.GetPower()
    try: gpu_processes = Monitor.GetGPUProcess() if 'adu' in self.info.name else {}
    except: gpu_processes = {}
    
    freqs = psutil.cpu_freq(percpu=True)
    for i, freq in enumerate(freqs):
      while len(self.info.cpu_freq) <= i: self.info.cpu_freq.append(0)
      self.info.cpu_freq[i] = freq.current
    if self.tick % 10 == 1:
      self.processes = []
      for p in psutil.process_iter():
        if 'sleep' in p.name(): continue
        if 'tbox' in self.info.name:
          if p.name() not in ['crload', 'python3', 'python', 'crpilot', 'audio_component']: continue
        self.processes.append(p)
    for p in self.processes:
      with p.oneshot():
        to = SystemMonitor.Process()
        try: to.name = p.name()
        except: continue
        to.cpu_percent = p.cpu_percent()
        if to.cpu_percent < 4.0: continue
        to.create_time = p.create_time()
        to.cpu_num = p.cpu_num()
        t = p.cpu_times()
        to.cpu_times.user = t.user
        to.cpu_times.system = t.system
        to.cpu_times.children_user = t.children_user
        to.cpu_times.children_system = t.children_system
        to.process_pid = p.pid  # TODO: 增加进程PID
        process_gpu_info = gpu_processes.get(p.pid)
        if process_gpu_info: to.gpu_mem, to.gpu = process_gpu_info # TODO: 增加进程gpu显存值，进程所属gpu编号
        try: to.cpu_times.iowait = t.iowait
        except: pass
        t = p.memory_info()
        to.memory_info.rss = t.rss
        to.memory_info.vms = t.vms
        to.memory_info.shared = t.shared
        to.memory_info.text = t.text
        to.memory_info.lib = t.lib
        to.memory_info.data = t.data
        to.memory_info.dirty = t.dirty
        t = p.num_ctx_switches()
        to.num_ctx_switches.voluntary = t.voluntary
        to.num_ctx_switches.involuntary = t.involuntary
        try:
          t = p.io_counters()
          to.io_counters.read_count = t.read_count
          to.io_counters.write_count = t.write_count
          to.io_counters.read_bytes = t.read_bytes
          to.io_counters.write_bytes = t.write_bytes
          to.io_counters.read_chars = t.read_chars
          to.io_counters.write_chars = t.write_chars
        except: pass
        try:
          for i in p.cmdline(): 
            try: to.cmdline.append(i)
            except: to.cmdline.add().CopyFrom(i)
        except: pass
        ps.append(to)
    ps = sorted(ps, key=lambda x: -x.cpu_percent)
    del self.info.processes[:]
    for p in ps: 
      try: self.info.processes.append(p)
      except: self.info.processes.add().CopyFrom(p)

    uptime, idle = [float(i) for i in open('/proc/uptime', 'r').read().split()]
    self.info.uptime = uptime
    mem = open('/proc/meminfo', 'r').read().split()
    self.info.ram.total, self.info.ram.free = float(mem[1]) / 1024, float(mem[7]) / 1024
    
    try:
      gpus = self.get_gpu()
      self.gpu_error_report = False
    except Exception as e:
      gpus = []
      if self.gpu_error_report == False:
        pycrmw.Stub('GPUFall', f'{e}')
        self.gpu_error_report = True
    while len(self.info.gpu) < len(gpus): gpu = self.info.gpu.add()
    for i in range(len(gpus)):
      gpu = self.info.gpu[i]
      temp, ram_total, ram_free, gpu_util = gpus[i]
          
      gpu.temp, gpu.ram.total, gpu.ram.free, gpu.util = temp, ram_total, ram_free, gpu_util
    
    if self.tick % 10 != 1: return
    self.GetTemperatures()
    self.GetROM()
    self.GetNetStatistics()
    self.GetTransfers()

  def GetTransfers(self): # 获取I/O速率相关
    if self.info.name in ["tboxhub", "sensorhub6-5"]:
      base_path = "/sys/devices/platform/fe150000.pcie/"
      for filename in ["pcie_speed", "cam_speed", "vnet_speed", "test_speed"]:
        try:
          with open(os.path.join(base_path, filename), 'r') as f:
            for line in f:
              # 使用正则表达式匹配dma通道和速度值
              match = re.match(r'dma-(\d+):\s+tx\s+(\d+(?:\.\d+)?)\s+MB/s(?:\s+rx\s+(\d+(?:\.\d+)?)\s+MB/s)?', line)
              if match:
                dma_channel = int(match.group(1))
                self.info.transfers[f'{filename}_{dma_channel}_tx'] = float(match.group(2))
                if filename != "cam_speed": 
                  self.info.transfers[f'{filename}_{dma_channel}_rx'] = float(match.group(3))
        except FileNotFoundError:
          pycrmw.log_error(f"Warning: File {filename} not found")
        except Exception as e:
          pycrmw.log_error(f"Error reading {filename}: {str(e)}")

  def GetTemperatures(self): # 硬件相关适配
    try:
      for name, v in psutil.sensors_temperatures(fahrenheit=False).items():
        for shwtemp in v:
          self.info.temperatures[f'{name}-{shwtemp.label}'] = shwtemp.current
      
      if self.get_cpu_temp == None:
        self.get_cpu_temp = lambda: math.nan
        if os.path.exists('/sys/class/thermal/thermal_zone0/temp'):
          self.get_cpu_temp = lambda: int(open('/sys/class/thermal/thermal_zone0/temp', 'r').read().strip()) / 1000.0
        if self.get_cpu_temp() < -20 and os.path.exists('/sys/class/thermal/thermal_zone1/temp'):
          self.get_cpu_temp = lambda: int(open('/sys/class/thermal/thermal_zone1/temp', 'r').read().strip()) / 1000.0
      self.info.cpu_temp = self.get_cpu_temp()
      self.UpdateSsdTemp()
      
      if 'adu5' in self.info.name:
        if len(self.ipmi_sensors) == 0:
          ipmi_sensors = [i.split('|')[0].strip() for i in custom_run('ipmitool sensor').split('\n')]
          self.ipmi_sensors = [i for i in ipmi_sensors if 'Temp' in i and ('VRM' in i or 'DIMM' in i or 'Peripheral' in i)]
        if len(self.ipmi_sensors): lines = custom_run('ipmitool sensor get ' + ' '.join([f'"{i}"' for i in self.ipmi_sensors]))
        else: lines = ''
        sensor = None
        for i in lines.split('\n'):
          if 'Sensor ID' in i: sensor = i.split(':')[-1].strip().split(' ')[0]
          if sensor != None and 'Reading' in i: self.info.temperatures[sensor] = int(i.split(':')[-1].strip().split(' ')[0])
      if 'tbox6' in self.info.name or 'hub6-5' in self.info.name or 'tbox7' in self.info.name:
        temp_file = '/sys/bus/iio/devices/iio:device0/in_temp0_raw'
        if os.path.exists(temp_file):
          self.info.temperatures['board'] = (float(open(temp_file, 'r').read())/2**20)*200-50
        humi_file = '/sys/bus/iio/devices/iio:device0/in_humidityrelative0_raw'
        if os.path.exists(humi_file):
          v = float(open(humi_file, 'rb').read())/2**20*100
          if len(self.info.humidity) == 0: self.info.humidity.append(v)
          else: self.info.humidity[0] = v
    except Exception as e:
      pycrmw.log_error(f'GetTemperatures failed: {e}')

  def UpdateSsdTemp(self):
    temps = {}
    temps.update(self.CollectNvmeTemp())
    temps.update(self.CollectSataTemp())
    for name, value in temps.items():
      self.info.temperatures[name] = value

  def CollectNvmeTemp(self):
    temps = {}
    nvme_cli = shutil.which('nvme')
    if nvme_cli == None:
      for candidate in ['/usr/sbin/nvme', '/sbin/nvme']:
        if os.path.exists(candidate):
          nvme_cli = candidate
          break
    if nvme_cli == None:
      return temps
    devices = sorted(set(glob.glob('/dev/nvme*')))
    for device in devices:
      base = os.path.basename(device)
      if not re.match(r'^nvme\d+(n\d+)?$', base): continue
      output = custom_run(f'{nvme_cli} smart-log {device}')
      if not output: continue
      for line in output.split('\n'):
        lower_line = line.lower()
        if 'temperature' not in lower_line: continue
        if 'temperature_sensor' in lower_line: continue
        match = re.search(r'(-?\d+)', line)
        if match:
          temps[f'ssd-{base}'] = float(match.group(1))
        break
    return temps

  def CollectSataTemp(self):
    temps = {}
    smartctl = shutil.which('smartctl')
    if smartctl == None:
      for candidate in ['/usr/sbin/smartctl', '/sbin/smartctl']:
        if os.path.exists(candidate):
          smartctl = candidate
          break
    if smartctl == None:
      return temps
    try:
      block_devices = os.listdir('/sys/block')
    except:
      return temps
    for name in sorted(block_devices):
      if not re.match(r'^sd[a-z]+$', name): continue
      device_path = f'/dev/{name}'
      output = custom_run(f'{smartctl} -A {device_path}')
      if not output: continue
      for line in output.split('\n'):
        if 'temperature' not in line.lower(): continue
        parts = line.split()
        temp_value = None
        for token in reversed(parts):
          if re.match(r'^-?\d+$', token):
            temp_value = float(token)
            break
        if temp_value != None:
          temps[f'ssd-{name}'] = temp_value
          break
    return temps

  def GetROM(self):
    du = psutil.disk_usage('/')
    self.info.rom.free = du.free / 1024 / 1024
    self.info.rom.total = du.total / 1024 / 1024

    if os.path.exists('/data'):
      du = psutil.disk_usage('/data')
      self.info.data.free = du.free / 1024 / 1024
      self.info.data.total = du.total / 1024 / 1024
    if os.path.exists('/traceback'):
      du = psutil.disk_usage('/traceback')
      self.info.traceback.free = du.free / 1024 / 1024
      self.info.traceback.total = du.total / 1024 / 1024
  def GetNetStatistics(self): # 硬件相关适配
    if 'tbox' in self.info.name:
      if len(self.info.ltes) < 1: self.info.ltes.add()
      try:
        self.info.ltes[0].rx_bytes = int(open('/sys/class/net/usb0/statistics/rx_bytes', 'r').read().strip())
        self.info.ltes[0].tx_bytes = int(open('/sys/class/net/usb0/statistics/tx_bytes', 'r').read().strip())
        if self.tick % 100 == 1:
          pycrmw.StubVariable("LTETXBytes", self.info.ltes[0].tx_bytes)
          pycrmw.StubVariable("LTERXBytes", self.info.ltes[0].rx_bytes)
      except: pass
  def GetPower(self): # 硬件相关适配
    if 'tbox6' in self.info.name or 'tbox7' in self.info.name:
      adc = '/sys/bus/iio/devices/iio:device1/in_voltage0_raw'
      if not os.path.exists(adc): return
      if 'tbox7' in self.info.name:
        vollist = [int(open(adc, 'rb').read())*1.8/4095 for i in range(5)]
      else:
        vollist = [int(open(adc, 'rb').read())/2048*3.3*16+0.2 for i in range(5)]
      vollist.sort()
      vbus = vollist[2]
      if len(self.info.power) == 0: power = self.info.power.add()
      else: power = self.info.power[0]
      power.power_name = self.info.name
      power.vin = vbus
    elif 'adu' in self.info.name:
      status = self.adu_power.Status()
      if status == None: return
      if isinstance(status, dict): status = [status]
      for i in range(0, len(status)):
        if len(self.info.power) <= i: power = self.info.power.add()
        else: power = self.info.power[i]
        power.power_name = status[i]['name']
        power.vin =  status[i]['vin']
        power.vout =  status[i]['vout']
        power.iout =  status[i]['iout']
        power.temperature =  status[i]['temp']
        power.power =  status[i]['power']

class NetLinkListen:
  RTMGRP_LINK = 1
  RTM_NEWLINK = 16
  IFLA_OPERSTATE = 16
  IF_OPER_DOWN = 2
  IF_OPER_UP = 6
  def __init__(self):
    self.nl_socket = socket.socket(socket.AF_NETLINK, socket.SOCK_RAW, socket.NETLINK_ROUTE)
    self.nl_socket.bind((0, self.RTMGRP_LINK))
    self.task = threading.Thread(target=self.RunFovever)
    self.task.start()
  def __del__(self):
    self.nl_socket.close()
  def RunFovever(self):
    while pycrmw.IsOK(): 
      try: self.Proc()
      except Exception as e:
        pycrmw.log_error(f'NetLinkListen {e}')
        time.sleep(1)
  def Proc(self):
    data = self.nl_socket.recv(65535)
    notify = {}
    while len(data) > 16:
      nlmsg_len, nlmsg_type, _, _, _ = struct.unpack("<LHHLL", data[0:16])
      if nlmsg_type == self.RTM_NEWLINK:
        ifi_family, _, ifi_type, ifi_index, ifi_flags, ifi_change = struct.unpack("<BBHIII", data[16:32])
        attribute_pos = 32
        while attribute_pos < nlmsg_len - 32:
          rta_len, rta_type, state = struct.unpack("<HHB", data[attribute_pos:attribute_pos+5])
          if rta_len == 0: break
          if rta_type == self.IFLA_OPERSTATE:
            if_name = socket.if_indextoname(ifi_index)
            if 'eno' in if_name or 'eth' in if_name or 'enp' in if_name:
              if state == self.IF_OPER_UP: notify[if_name] = True
              elif state == self.IF_OPER_DOWN: notify[if_name] = False
          attribute_pos += int((rta_len + 3) / 4) * 4
      data = data[nlmsg_len:]
    for k, v in notify.items():
      if v == False and float(open('/proc/uptime').read().split(' ')[0]) > 240: pycrmw.Stub("NetUnexpectedDown", k)

class MonitorPush:
  def __init__(self):
    self.sequence = 1
    self.data = SimpleMonitor()
    self.data.sequence = self.sequence
    
    self.s = Stubs()
    self.s.hostname = socket.gethostname()
    self.s.version = pycrmw.Version()
    self.e = self.s.events.add()
    self.e.name = '$Monitor'
    self.rpc = pycrmw.ServiceFind('LoggerStub', Stubs, Stubs)
    self.lock = threading.Lock()
    
  @report_stub
  def Push(self, m: SystemMonitor):
    try:
      self.lock.acquire()
      samples = self.data.systems[m.name]
      if len(samples.samples) == 0: samples.timestamp = m.timestamp
      sample = samples.samples.add()
      if m.cpu.total >= 0.1:
        sample.cpu = int(1000 - 1000*m.cpu.free / m.cpu.total)
        sample.ram = int(1000 - 1000*m.ram.free / m.ram.total)
        sample.cpu_temp = int(m.cpu_temp) if not math.isnan(m.cpu_temp) else 0
        sample.rootfs = int(m.rom.free)
        sample.data = int(m.data.free)
        for i in m.cpu_freq: sample.cpu_freq.append(int(i / 100)) if i > 100 else sample.cpu_freq.append(int(i * 1024 / 100))
        for gpu in m.gpu: sample.gpu.append(int(gpu.util * 1000))
        for gpu in m.gpu: sample.gpu_temp.append(int(gpu.temp))

      for power in m.power:
        sample.power_voltage.append(int(power.vin * 1000))
        sample.power_current.append(int(power.iout * 1000))
        sample.power_temp.append(int(power.temperature))
      if len(m.humidity): sample.humidity = int(m.humidity[-1])
      
      for i in m.processes:
        if len(i.cmdline) == 0: continue
        if 'python' in i.cmdline[0]:
          to = self.data.process[' '.join(i.cmdline[:3])]
        else:
          to = self.data.process[i.cmdline[0]]
        to.cpu = int(i.cpu_percent)
        to.ram = int(i.memory_info.rss / 1024)
        if i.gpu_mem:
          to.gpu = int(i.gpu)
          to.gpu_mem = int(i.gpu_mem)

        to.process_pid = i.process_pid
      for k, v in m.temperatures.items(): self.data.temps[k] = int(v)
      for k, v in m.transfers.items(): self.data.transfer[k] = int(v)
    finally:
      if samples.timestamp * 1e-9  + 119.5 < time.time() :
        pycrmw.log_info(f'samples.size:{len(samples.samples)}') 
        self.Publish()
      self.lock.release()
  def Publish(self):
    self.data.timestamp = int(time.time() * 1e9)
    self.sequence = self.sequence + 1
    self.e.timestamp = self.data.timestamp
    self.e.attachment = self.data.SerializeToString()
    self.rpc(self.s)
    self.data = SimpleMonitor()
    self.data.sequence = self.sequence

class AdditionalMonitor():
  def __init__(self, name = 'AdditionalMonitor', push = None):
    if 'tbox' not in socket.gethostname(): return
    self.name, self.push = name, push
    self.task = threading.Thread(target = self.__udpate)
    self.task.start()

  def __udpate(self):
    try:
      pycrmw.log_info(f'AdditionalMonitor {self.name} running')
      power = None
      while pycrmw.IsOK():
        time.sleep(10)
        ret = custom_run('ipmiutil fru -N 192.168.1.254 -U cowa -P Cowa2022 -F lan2')
        if 'Supermicro' in ret: break
        elif 'AST2500' in ret:
          pycrmw.log_info(f'AdditionalMonitor {self.name} AST2500 Found')
          if os.system('command -v ipmitool > /dev/null') == 0: power = Pmbus6()
          break
      info = SystemMonitor()
      info.sequence = 1
      info.name = self.name
      p = info.power.add()
      while pycrmw.IsOK() and power:
        time.sleep(1)
        status = power.Status()
        if status == None: continue
        p.power_name = 'adu-' + status['name']
        p.vin, p.vout, p.iout, p.temperature, p.power = status['vin'], status['vout'], status['iout'], status['temp'], status['power']
        info.sequence, info.timestamp = info.sequence + 1, int(time.time() * 1e9)
        if self.push: self.push.Push(info)
      pycrmw.log_info(f'AdditionalMonitor {self.name} exit')
    except Exception as e: pycrmw.log_error(f'AdditionalMonitor update error:{str(e)}')
    
if __name__ == "__main__":
  m = Monitor()
  pycrmw.Init(sys.argv)
  node = pycrmw.Node(sys.argv[0])
  pub = node.CreateWriter('/monitor/system', SystemMonitor)
  start_time = time.monotonic()
  cpu_overload_tick = 0
  if 'adu' in m.info.name:
    listen = NetLinkListen()
  space_free = 0
  push = MonitorPush()
  interval_time = time.monotonic()
  if 'adu' in m.info.name or 'ADU' in m.info.name:
    interval = 1
  else:
    interval = 3
    
  bmc_power = AdditionalMonitor('adu-power', push)
  while pycrmw.IsOK():
    try:
      m.update()
      pub.Write(m.info)
    except: 
      pycrmw.log_error(traceback.format_exc())
    try: push.Push(m.info)
    except: pycrmw.log_error(traceback.format_exc())

    try:
      pycrmw.StubVariable("CPUPercentage", int(100 - 100 * m.info.cpu.free / m.info.cpu.total))
      # pycrmw.StubVariableF("RAMPercentage", int(100 - 100 * m.info.ram.free / m.info.ram.total))
      if m.info.ram.total < 1024 * 4: mem_th = 128 # tbox等内存预留较小
      elif m.info.ram.total < 1024 * 8: mem_th = 256
      else: mem_th = m.info.ram.total * 0.1
      if m.info.ram.free < mem_th or m.info.cpu.free <  m.info.cpu.total * 0.15: 
        top = custom_run('top -b -n1')
      else: top = ''
      
      pycrmw.StubTrigger('MemoryInsufficient', m.info.ram.free < mem_th, top)
      if time.monotonic() > start_time + 60: # 开机的前1分钟屏蔽cpu 占用，因为此阶段天然CPU占用过高
        if m.info.cpu.free <  m.info.cpu.total * 0.15: cpu_overload_tick += 1
        else: cpu_overload_tick = 0
        pycrmw.StubTrigger('CPUInsufficient', cpu_overload_tick > 10, top)
        start_time = time.monotonic() + 240 # 5分钟内不再上报
      
      pycrmw.StubTrigger('RootfsInsufficient', m.info.rom.free < m.info.rom.total * 0.1, f'{m.info.rom}')
      pycrmw.StubTrigger('DataInsufficient', m.info.data.free < 10000 and m.info.data.total > 100000, f'{m.info.data}')
      if m.info.rom.free < m.info.rom.total * 0.15:
        if space_free == 0:
          pycrmw.log_error('limit log size')
          os.system('journalctl --vacuum-size=200M')
          os.system('logrotate -f /etc/logrotate.conf')
        elif space_free < 7:
          pycrmw.log_error(f'rm old log old then {7 - space_free} day')
          os.system(f'find /var/log/ -type f -name "*.log.*" -mtime +{7 - space_free}' + ' -exec rm {} \\;')
        space_free += 1
    except:
      import traceback
      pycrmw.log_error(f'Stub error {traceback.format_exc()}')

    # 时间间隔
    if time.monotonic() - interval_time < interval:
      try:
          time.sleep(interval - time.monotonic() + interval_time)
      except:
        pycrmw.log_error(f'sleep faild: {traceback.format_exc()}')
    interval_time = time.monotonic()