#!/usr/bin/env python3

import sys
import time
import subprocess
import socket
import random

class CommError(Exception):
    pass

class FipUrvDrv:
    def __init__(self, trace=False):
        if False:
            self.proc = subprocess.Popen(["./mfip_urv", "-p"],
                                         universal_newlines=True,
                                         bufsize=0,
                                         stdin=subprocess.PIPE,
                                         stdout=subprocess.PIPE)
            self.sock = None
        else:
            self.proc = None
            self.sock = socket.create_connection(("127.0.0.1", 2000))

        self.trace = trace
        self.nbr_comm_err = 0
        self.nbr_ping_err = 0
        self.nbr_flash_err = 0

    def read_reply(self):
        l = ''
        res = []
        while True:
            if self.sock:
                c = self.sock.recv(1).decode('ascii')
            else:
                c = self.proc.stdout.read(1)
            if c == 0:
                # EOF
                break
            if c == '\n':
                if len(l) > 2 and l[:2] == '# ':
                    break
                res.append(l)
                l = ''
            else:
                l += c
        if self.trace:
            sys.stderr.write("Recv: {} + {}\n".format(res, l))
        if l != "# OK":
            print("ERROR: comm, status: {}".format(l))
            self.nbr_comm_err += 1
            raise CommError
        return res

    def send_cmd(self, cmd):
        if self.trace:
            sys.stderr.write("Send: {}\n".format(cmd))
        if self.sock:
            self.sock.send(cmd.encode('ascii') + b'\n')
        else:
            self.proc.stdin.write(cmd + '\n')


    def cmd(self, cmd):
        self.send_cmd(cmd)
        return self.read_reply()

    def disp(self, res):
        for l in res:
            print(l, end='')

    def close(self):
        if self.proc:
            self.proc.stdin.close()
            self.read_reply()
            self.proc.wait()
        else:
            self.sock.close()


def get_pow_vc(s, page):
    s.cmd("wmon page {}".format(page))
    vout = s.cmd("mon -s vout")
    iout = s.cmd("mon -s iout")
    return (vout[0], iout[0])

def get_pow_v(s, page):
    s.cmd("wmon page {}".format(page))
    vout = s.cmd("mon -s vout")
    return vout[0]


def get_cmd(s, cmd):
    res = s.cmd("mon -s {}".format(cmd))
    return res[0]

def show_cmd(s, cmd, name):
    res = get_cmd(s, cmd)
    print("{}:  {}".format(name, res))

REGS = 0x100000
SPI = REGS + 0xe0

def spi_init(s):
    # Set divider (1Mhz)
    s.cmd("write 0x{:x} 20".format(SPI + 0x14))

def spi_cmd(s, cmd):
    """Send cmd to spi.  cmd is a list of bytes"""
    l = len(cmd)
    assert 0 < l <= 8

    # Set ctrl char-len=5*8=0x28, tx_neg=1, rx_neg=0
    s.cmd("write 0x{:x} 0x{:04x}".format(SPI + 0x10, 0x400 + l * 8))

    # Bytes to be sent
    ra = SPI + ((l - 1) // 4) * 4

    w = 0
    r = ra
    for i in range(l):
        off = (l - 1 - i) & 3
        #print("  l={}, i={}, off={}".format(l, i, off))
        w |= cmd[i] << (off * 8)
        if off == 0 or i == l - 1:
            s.cmd("write 0x{:x} 0x{:x}".format(r, w))
            r -= 4
            w = 0

    # Set SS=1
    s.cmd("write 0x{:x} 1".format(SPI + 0x18))

    # Go
    s.cmd("write 0x{:x} 0x{:04x}".format(SPI + 0x10, 0x500 + l * 8))

    # Read
    res = []
    r = ra
    for i in range(l):
        off = (l - 1 - i) & 3
        if i == 0 or off == 3:
            w = s.cmd("read 0x{:x}".format(r))
            r -= 4
            w = int(w[0], 0)
        v = (w >> (off * 8)) & 0xff
        res.append(v)

    # Set SS=0
    s.cmd("write 0x{:x} 0".format(SPI + 0x18))

    return res

def flash_id(s):
    spi_init(s)

    res = spi_cmd(s, [0x05, 0])
    print("Flash status: {:02x}".format(res[1]))

    res = spi_cmd(s, [0x9f, 0, 0, 0, 0])
    print("Flash id: {:02x}{:02x}{:02x}{:02x}".format(
        res[1], res[2], res[3], res[4]))

    # Read
    for i in range(10):
        res = spi_cmd(s, [0x03, 0, i, 0, 0, 0, 0, 0])
        print("Flash 0x{:04x}: {:02x} {:02x} {:02x} {:02x}".format(
            i << 8, res[4], res[5], res[6], res[7]))

def flash_wait_write(s):
    while True:
        res = spi_cmd(s, [0x05, 0])
        print("Flash status: {:02x}".format(res[1]))
        if (res[1] & 0x20) != 0:
            print("ERROR: flash write/erase")
            s.nbr_flash_err += 1
            break
        if (res[1] & 1) == 0:
            break

def flash_prg(s, pg, off):
    print("Write flash at page={:02x} off={:02x}".format(pg, off))

    # Write enable
    spi_cmd(s, [0x06])

    # Write
    npg = (pg + 1) & 0xff
    spi_cmd(s, [0x02, 0x00, pg, off, pg, (~pg) & 0xff, npg, (~pg) & 0xff])

    flash_wait_write(s)

    # Write disable
    spi_cmd(s, [0x04])


def flash_full_erase(s):
    print("Erasing flash")

    # Write enable
    spi_cmd(s, [0x06])

    # Erase
    spi_cmd(s, [0x60])

    flash_wait_write(s)

def flash_page_erase(s, pg):
    print("Erasing flash page 0x{:02x}".format(pg))

    # Write enable
    spi_cmd(s, [0x06])

    # Page erase
    spi_cmd(s, [0x81, 0x00, pg, 0x00])

    flash_wait_write(s)

def flash_find_free(s):
    """Assuming flash pages are written consecutively, find the first erased
    one"""
    first = 0
    last = 255

    print("Find first free page:")
    while True:
        mid = (first + last) // 2
        res = spi_cmd(s, [0x03, 0, mid, 0, 0, 0])
        print("Page {}: first byte is {:02x} {:02x} (first:{}, last:{})".format(
            mid, res[4], res[5], first, last))
        if res[4] == 0xff and res[5] == 0xff:
            # Page is free
            if mid == first:
                return mid
            last = mid - 1
        else:
            # Page is busy
            if mid == last:
                if last == 255:
                    return -1
                return last + 1
            first = mid + 1

def show_psu(s):
    psu = s.cmd("psu")
    s.cmd("i2c 2")
    print("********** PSU 0 ******************")
    if psu[0] == "PSU 0: presence: 0":
        print("Not present")
    elif psu[0] == "PSU 0: presence: 1":
        print("TODO")
    else:
        print("unhandled")
    print("********** PSU 1 ******************")
    if psu[1] == "PSU 1: presence: 0":
        print("Not present")
    elif psu[1] == "PSU 1: presence: 1":
        show_cmd(s, "-t 0x18 vout", "Vout")
        show_cmd(s, "-t 0x18 iout", "Iout")
        show_cmd(s, "-t 0x18 temp-1", "Temp 1")
        show_cmd(s, "-t 0x18 temp-2", "Temp 2")
    else:
        print("unhandled")

def show_boards(s):
    print("********** Configuration *********")
    s.cmd("i2c 1")
    print("SLOTS: 01 02 03 04 05 06 07 08")
    print("      ", end='')
    for i in range(8):
        res = s.cmd("i2c-id -s {}".format(i))
        print(" {}".format(res[0]), end='')
    print()

def show_sysmon(s):
    s.cmd("i2c 0")
    t = time.time()
    ts = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(t))
    rev = get_cmd(s, "revision")
    print("******** System board (revision {}) ********".format(rev))
    print("time: {}".format(ts))
    p12 = get_pow_vc(s, 0)
    print("P12V:   {}   {}".format(p12[0], p12[1]))
    p1_2 = get_pow_v(s, 1)
    print("P1.2V:  {}".format(p1_2))
    p3_3 = get_pow_v(s, 2)
    print("P3.3V:  {}".format(p3_3))
    pperi = get_pow_v(s, 3)
    print("Vperi:  {}".format(pperi))
    tamb = get_cmd(s, "temp-1")
    print("Temp amb:  {}".format(tamb))
    tldo = get_cmd(s, "temp-2")
    print("Temp LDO:  {}".format(tldo))
    tfpga = get_cmd(s, "temp-3")
    print("Temp FPGA: {}".format(tfpga))
    print("CSV: {tm},{p12v},{p12i},{p1_2},{p3_3},{tamb},{tldo},{tfpga},{ncom_err},{nflash_err},{nping_err}".format(
        tm=ts,
        p12v=p12[0].rstrip(' V'),
        p12i=p12[1].rstrip(' A'),
        p1_2=p1_2.rstrip(' V'),
        p3_3=p3_3.rstrip(' V'),
        tamb=tamb.rstrip(' C'),
        tldo=tldo.rstrip(' C'),
        tfpga=tfpga.rstrip(' C'),
        ncom_err=s.nbr_comm_err,
        nflash_err=s.nbr_flash_err,
        nping_err=s.nbr_ping_err))

rnd = random.Random()

def do_ping(s):
    v = rnd.randint(0, (1<<32) - 1)
    print("Ping with value {:08x}".format(v))
    s.cmd("write 0x107fc 0x{:08x}".format(v))
    w = s.cmd("read 0x107fc")
    rv = int(w[0], 0)
    if rv != v:
        print("ERROR: ping (write {:08x}, read {:08x})".format(v, rv))
        s.nbr_ping_err += 1

def extract(s, left, right=None):
    if len(s) < len(left):
        sys.stderr.write("extract ERR1: '{}' from '{}'\n".format(left, s))
        return "ERR1"
    if s[:len(left)] != left:
        sys.stderr.write("extract ERR2: '{}' from '{}'\n".format(left, s))
        return "ERR2"
    s = s[len(left):]
    if right is None:
        return s
    if len(s) < len(right):
        sys.stderr.write("extract ERR3: '{}' from '{}'\n".format(left, s))
        return "ERR3"
    if s[-len(right):] != right:
        sys.stderr.write("extract ERR4: '{}' from '{}'\n".format(left, s))
        return "ERR4"
    return s[:-len(right)]

def do_extract_recovery(s):
    p = s.cmd("stat-regs")
    nrecv = int(extract(p[4], "nbr recovery: "), 0)
    nrst = int (extract(p[5], "nbr resets: "), 0)
    last = int(extract(p[6], "last reset: "), 0)
    return (nrecv, nrst, last)

def do_extract_reset_cause(s):
    p = s.cmd("supervisor")
    return int(extract(p[0], "reset cause: "), 0)

def do_self_tests_recovery(s):
    print("* Test CPU recovery")
    nrecv, nrst, _ = do_extract_recovery(s)
    for i in range(3):
        rst_cause = do_extract_reset_cause(s)
        print("Fault on cpu {}".format(i))
        s.cmd("cpu-fault {}".format(i))
        nrecv1, nrst1, last = do_extract_recovery(s)
        if nrecv1 != nrecv + 1:
            print("Bad nbr recovery: read {}, expect {}".format(
                nrecv1, nrecv + 1))
        else:
            nrecv += 1
        if nrst1 != nrst + 1:
            print("Bad nbr reset: read {}, expect {}".format(
                nrst1, nrst + 1))
        else:
            nrst += 1
        if last != (1 << i):
            print("Bad last reset: read {}, expect {}".format(last, 1 << i))
            cause = do_extract_reset_cause(s)
            if cause != 1:
                print("bad reset cause, read {}, expect 1".format(cause))

def do_extract_dram_ecc(s):
    p = s.cmd("supervisor")
    n_ecc_se = int(extract(p[11], "DRAM corrected errors by scrubber: "), 0)
    n_ecc_de = int(extract(p[13],
                           "DRAM double errors detected in last cycle: "), 0)
    return (n_ecc_se, n_ecc_de)

def do_self_tests_dram(s):
    print("* Test DRAM ecc")
    n_ecc_se, n_ecc_de = do_extract_dram_ecc(s)
    print("se: {}, de: {}".format(n_ecc_se, n_ecc_de))
    if n_ecc_de != 0:
        print("Already data ECC errors")
    s.cmd("write-ecc 0x40100 0x123 1")
    n_ecc_se1, n_ecc_de1 = do_extract_dram_ecc(s)
    print("se: {}, de: {}".format(n_ecc_se1, n_ecc_de1))
    if n_ecc_se1 != n_ecc_se + 1 or n_ecc_de1 != 0:
        print("bad number of ECC errors")
    n_ecc_se = n_ecc_se1
    s.cmd("write-ecc 0x40104 0x456 6")
    n_ecc_se1, n_ecc_de1 = do_extract_dram_ecc(s)
    print("se: {}, de: {}".format(n_ecc_se1, n_ecc_de1))
    if n_ecc_se1 != n_ecc_se or n_ecc_de1 != 1:
        print("bad number of ECC errors")
    s.cmd("read 0x40114")
    cause = do_extract_reset_cause(s)
    if cause != 4:
        print("bad reset cause, read {}, expect 4".format(cause))
    s.cmd("write-ecc 0x40104 0 0")

def do_extract_iram_ecc(s):
    p = s.cmd("supervisor")
    n_ecc_se = int(extract(p[6], "IRAM corrected errors by scrubber: "), 0)
    n_ecc_de = int(extract(p[8],
                           "IRAM double errors detected in last cycle: "), 0)
    return (n_ecc_se, n_ecc_de)

def do_self_tests_iram(s):
    print("* Test IRAM ecc")
    n_ecc_se, n_ecc_de = do_extract_iram_ecc(s)
    print("se: {}, de: {}".format(n_ecc_se, n_ecc_de))
    if n_ecc_de != 0:
        print("Already insn ECC errors")
    s.cmd("restart-cfg 7")
    print("crash...")
    s.cmd("write-ecc 0x40114 0x456 6")
    s.cmd("read 0x40114")
    # Will timeout
    cause = do_extract_reset_cause(s)
    if cause != 4:
        print("bad reset cause, read {}, expect 4".format(cause))
    s.cmd("restart-cfg 3")
    n_ecc_se1, n_ecc_de1 = do_extract_iram_ecc(s)
    print("se: {}, de: {}".format(n_ecc_se1, n_ecc_de1))
    if n_ecc_se1 != n_ecc_se + 2 or n_ecc_de1 != 2:
        print("Error in IRAM ecc")
    # Crash too
    s.cmd("read 0xfffc")
    cause = do_extract_reset_cause(s)
    if cause != 2:
        print("bad reset cause, read {}, expect 2".format(cause))

def do_extract_monimod_cycle(s):
    p = s.cmd("monimod-read")
    return int(extract(p[0], "Cycle: "), 0)

def do_extract_monimod_reset(s):
    p = s.cmd("stat-regs")
    return int(extract(p[0], "nbr monimod reset: "), 0)

def do_self_tests_monimod_scan(s, cnt=20):
    cyc = do_extract_monimod_cycle(s)
    ntry = cnt
    while ntry > 0:
        ncyc = do_extract_monimod_cycle(s)
        if ncyc != cyc:
            break
        ntry -= 1
    if ntry == 0:
        print("no monimod scan")

def do_self_tests_monimod(s):
    print('* Test monimod')
    do_self_tests_monimod_scan(s)
    nrst = do_extract_monimod_reset(s)
    s.cmd("monimod-power off")
    do_self_tests_monimod_scan(s, 100)
    nrst1 = do_extract_monimod_reset(s)
    if nrst1 != nrst + 1:
        print("monimod reset expected")

def do_self_tests(s):
    do_self_tests_recovery(s)
#    do_self_tests_dram(s)
#    do_self_tests_iram(s)
#    do_self_tests_monimod(s)

class MyCSV:
    def __init__(self):
        self.csv = []
        self.prev = {}
        self.change = 'start'

    def add(self, name, val, test=True):
        self.csv.append((name, val))
        if test and not self.change and self.prev[name] != val:
            self.change = name

    def add_percnt(self, name, val, pcnt=5.0):
        self.csv.append((name, val))
        if not self.change:
            val = float(val)
            prev = float(self.prev[name])
            dlt = prev * pcnt / 100.0
            if dlt < 0.0:
                dlt = -dlt
            if val < prev - dlt or val > prev + dlt:
                self.change = name

    def get_header(self):
        return ','.join(t[0] for t in self.csv)

    def get_csv(self):
        return ','.join(str(t[1]) for t in self.csv)

    def get_change(self):
        return self.change

    def has_changed(self):
        return self.change is not None

    def update(self):
        self.prev = {t[0]:t[1] for t in self.csv}
        self.csv = []
        self.change = None

    def discard(self):
        self.csv = []
        self.change = None


def do_plc_supervisor(s, csv, verbose, num):
    p = s.cmd("plc {}".format(num))
    if verbose:
        print("plc{}: {}".format(num, p))
    csv.add('=plc{}='.format(num), '=plc{}='.format(num))
    csv.add('plc{}-reset'.format(num),
            extract(p[0], "plc reset:  ", "  # 1: sys-rst, 2: cpu-rst"))
    csv.add('plc{}-status'.format(num),
            extract(p[1], "plc status: ", "  # 1: fault"))
    p = s.cmd("plc-supervisor {}".format(num))
    if verbose:
        print("plc-supervisor{}: {}".format(num, p))
    csv.add('plc{}-reset-cause'.format(num),
            extract(p[0], "PLC reset cause: ",
                    "  # 1: cpu, 2: iram, 4: dram, 8: wd, 16: recov"))
    csv.add('plc{}-iram-se-corr'.format(num),
            extract(p[1], "PLC IRAM corrected errors: "))
    csv.add('plc{}-iram-scrub-cyc'.format(num),
            extract(p[2], "PLC IRAM scrub cycle number: "), False)
    csv.add('plc{}-iram-scrub-se'.format(num),
            extract(p[3], "PLC IRAM corrected errors by scrubber: "))
    csv.add('plc{}-iram-scrub-de'.format(num),
            extract(p[4], "PLC IRAM double errors detected in last cycle: "))
    csv.add('plc{}-dram-se-corr'.format(num),
            extract(p[5], "PLC DRAM corrected errors: "))
    csv.add('plc{}-dram-scrub-cyc'.format(num),
            extract(p[6], "PLC DRAM scrub cycle number: "), False)
    csv.add('plc{}-dram-scrub-se'.format(num),
            extract(p[7], "PLC DRAM corrected errors by scrubber: "))
    csv.add('plc{}-dram-scrub-de'.format(num),
            extract(p[8], "PLC DRAM double errors detected in last cycle: "))
    csv.add('plc{}-scrub-cfg'.format(num),
            extract(p[9], "PLC scrub config: "))
    csv.add('plc{}-cpu-fault'.format(num),
            extract(p[10], "PLC nbr cpu fault: "))

def run():
    s = FipUrvDrv(trace=False)
    # s.cmd("slots")

    verbose = False
    flag_dry = False
    flag_flash = False
    flag_iram_err = False
    flag_all = False
    flag_monitor = False
    flag_load = True
    iram_err_addr = 0
    iram_err_cnt = 0
    log = None
    count = 0

    for arg in sys.argv[1:]:
        if arg == '-v':
            verbose = True
        elif arg == '--dry' or arg == '-n':
            flag_dry = True
        elif arg == '--no-flash':
            flag_flash = False
        elif arg == '--iram-err':
            flag_iram_err = True
        elif arg == '-t':
            s.trace = True
        elif arg == '--no-load':
            flag_load = False
        elif arg == '--all':
            # Print all the CSV (otherwise, only on a change)
            flag_all = True
        elif arg == '--monitor':
            # 1/200 CSV
            flag_monitor = True
        elif arg[:2] == '-l':
            log = open(arg[2:], "a")
        else:
            print("unknown option: {}".format(arg))
            sys.exit(1)

    s.cmd("monimod-scan on")
    if flag_load:
        s.cmd("plc 0 reset")
        s.cmd("plc 0 halt")
        s.cmd("load 0 plc0_mbox_inc.elf")
        s.cmd("plc 1 reset")
        s.cmd("plc 1 halt")
        s.cmd("load 1 plc1_mbox_not.elf")
        s.cmd("read 0xff0")

    if flag_flash:
        flash_id(s)

    print("Self-tests sequence")
    do_self_tests(s)

    if flag_dry:
        return

    flag_erase = False

    # Find first free page
    if flag_flash:
        pg = flash_find_free(s)
        if pg < 0:
            print("No free page")
            flash_full_erase(s)
            pg = 0
            print("First free page: {}".format(pg))
        off = 0

    mbx1_exp = 0x12345678
    mbx0_exp = 0x23456789

    ping_err = 0
    mbx1_err = 0
    mbx0_err = 0

    first = True
    change = True
    prev = {}
    csv = MyCSV()
    while True:
        try:
            t = time.time()
            ts = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(t))
            ts += ".{}".format(int(t * 10) % 10)  # Append 10th of seconds
            # csv = [('time', ts)]
            if s.trace:
                sys.stderr.write("Date: {}\n".format(ts))
            p = s.cmd("ping")
            if verbose:
                print("Ping: {}".format(p))
            if p[0].find('!') != -1:
                ping_err += 1
            csv.add('ping', ping_err)
            p = s.cmd("fip-err")
            if verbose:
                print("fip-err: {}".format(p))
            csv.add('fip-err', extract(p[0], "fip errors: "))
            csv.add('comm-err', s.nbr_comm_err)
            p = s.cmd("system-read")
            if verbose:
                print("System: {}".format(p))
            csv.add('=sys=', '=sys=')
            csv.add('cycle', extract(p[0], "Cycle: "), False)
            p = p[3:]
            pr = p and p[0].startswith('P12V')
            csv.add_percnt('P12V', extract(p[0], "P12V: ", " V") if pr else 0)
            csv.add_percnt('P12A', extract(p[1], "P12V: ", " A") if pr else 0)
            csv.add_percnt('P1.2V', extract(p[2], "P1.2V: ", " V") if pr else 0)
            csv.add_percnt('P3.3V', extract(p[3], "P3.3V: ", " V") if pr else 0)
            csv.add_percnt('Vperi', extract(p[4], "Vperi: ", " V") if pr else 0)
            csv.add_percnt('Tamb', extract(p[5], "Tamb: ", " C") if pr else 0)
            csv.add_percnt('Tldo', extract(p[6], "Tldo: ", " C") if pr else 0)
            csv.add_percnt('Tfpga', extract(p[7], "Tfpga: ", " C") if pr else 0)
            if pr:
                p = p[8:]
            csv.add('=fan=', '=fan=')
            pr = p and p[0].startswith('FTP12V')
            csv.add_percnt('FTP12V', extract(p[0], "FTP12V: ", " V") if pr else 0)
            csv.add_percnt('FTP12A', extract(p[1], "FTP12V: ", " A") if pr else 0)
            csv.add_percnt('FTP5V', extract(p[2], "FTP5V: ", " V") if pr else 0)
            csv.add_percnt('FTtemp', extract(p[3], "FTtemp: ", " C") if pr else 0)
            csv.add_percnt('fan1', extract(p[4], "fan 1: ") if pr else 0)
            csv.add_percnt('fan2', extract(p[5], "fan 2: ") if pr else 0)
            csv.add_percnt('fan3', extract(p[6], "fan 3: ") if pr else 0)
            if pr:
                p = p[7:]
            pr = p and p[0].startswith('adc4')
            csv.add_percnt('adc4', extract(p[0], "ADC 4: ") if pr else 0)
            csv.add_percnt('adc5', extract(p[1], "ADC 5: ") if pr else 0)
            csv.add_percnt('adc6', extract(p[2], "ADC 6: ") if pr else 0)
            csv.add_percnt('adc7', extract(p[3], "ADC 7: ") if pr else 0)
            csv.add_percnt('adcT', extract(p[4], "adcT: ", " C") if pr else 0)
            csv.add('=sv=', '=sv=')
            p = s.cmd("supervisor")
            if verbose:
                print("supervisor: {}".format(p))
            csv.add('reset-cause', extract(p[0], "reset cause: "))
            csv.add('cpus-reset', extract(p[1], "cpus on reset: "))
            csv.add('iram-se-corr', extract(p[4], "IRAM corrected errors: "))
            csv.add('iram-scrub-cyc',
                    extract(p[5], "IRAM scrub cycle number: "), False)
            csv.add('iram-scrub-se',
                    extract(p[6], "IRAM corrected errors by scrubber: "))
            csv.add('iram-scrub-de',
                    extract(p[8], "IRAM double errors detected in last cycle: "))
            csv.add('dram-se-corr',
                    extract(p[9], "DRAM corrected errors: "))
            csv.add('dram-scrub-cyc',
                    extract(p[10], "DRAM scrub cycle number: "), False)
            csv.add('dram-scrub-se',
                    extract(p[11], "DRAM corrected errors by scrubber: "))
            csv.add('dram-scrub-de',
                    extract(p[13], "DRAM double errors detected in last cycle: "))
            csv.add('insn-bus-err',
                    extract(p[14], "Errors on cpu data bus: "))
            csv.add('data-bus-err',
                    extract(p[15], "Errors on cpu insn bus: "))
            csv.add('ram-scrub-cfg',
                    extract(p[16], "Scrubber config: "))
            csv.add('iram-ecc-addr',
                    extract(p[19], "IRAM ecc addr: "))
            csv.add('=sta=', '=sta=')
            p = s.cmd("stat-regs")
            if verbose:
                print("stat-regs: {}".format(p))
            csv.add('monimod-reset',
                    extract(p[0], "nbr monimod reset: "))
            csv.add('mm-fantray-reset',
                    extract(p[1], "nbr fantray reset: "))
            csv.add('ratopus-errs',
                    extract(p[2], "nbr ratopus errors: "))
            csv.add('min-watchdog',
                    extract(p[3], "minimum watchdog count: "))
            csv.add('nbr-recovery',
                    extract(p[4], "nbr recovery: "))
            csv.add('nbr-reset',
                    extract(p[5], "nbr resets: "))
            csv.add('last-reset',
                    extract(p[6], "last reset: "))
            csv.add('cycle-count',
                    extract(p[7], "cycle count: "), False),
            csv.add('fip-cmds',
                    extract(p[8], "fip commands: "), False)
            csv.add('fip-pad',
                    extract(p[9], "fip pad: "), False)
            if False and len(p) > 12:
                csv.add('fip-writes',
                        extract(p[12], "fip writes: "))
                csv.add('wr-addr',
                        extract(p[13], "write addr: "))

            do_plc_supervisor(s, csv, verbose, 0)

            csv.add('=mbx0=', '=mbx0=')
            p = s.cmd("mbox 0")
            if verbose:
                print("mbox 0: {}".format(p))
            mbx0_sta = extract(p[0], "mailbox status: ")
            mbx0_val = int(extract(p[1], "mailbox in value: "), 0)
            if mbx0_sta != '1':
                print("mailbox not ready")
                mbx0_err += 1
            else:
                if mbx0_val != mbx0_exp:
                    print("mailbox error: expect {:08x}, read {:08x}".format(
                        mbx0_exp, mbx0_val))
                    mbx0_err += 1
            # Always write so we are not stuck
            mbx0_nval = rnd.randint(0, (1<<32) - 1)
            s.cmd("mbox 0 0x{:08x}".format(mbx0_nval))
            mbx0_exp = (mbx0_nval + 1) & 0xffffffff
            csv.add('mbx0-err', mbx0_err)

            do_plc_supervisor(s, csv, verbose, 1)
            csv.add('=mbx1=', '=mbx1=')
            p = s.cmd("mbox 1")
            if verbose:
                print("mbox 1: {}".format(p))
            mbx1_sta = extract(p[0], "mailbox status: ")
            mbx1_val = int(extract(p[1], "mailbox in value: "), 0)
            if mbx1_sta != '1':
                print("mailbox not ready")
                mbx1_err += 1
            else:
                if mbx1_val != mbx1_exp:
                    print("mailbox error: expect {:08x}, read {:08x}".format(
                        mbx1_exp, mbx1_val))
                    mbx1_err += 1
            # Always write so we are not stuck
            mbx1_nval = rnd.randint(0, (1<<32) - 1)
            s.cmd("mbox 1 0x{:08x}".format(mbx1_nval))
            mbx1_exp = (~mbx1_nval) & 0xffffffff
            csv.add('mbx1-err', mbx1_err)

            csv.add('flash-err', s.nbr_flash_err)

            if first:
                if log:
                    log.write('time,' + csv.get_header() + '\n')
                print('time,' + csv.get_header())
                first = False

            line = ts + ',' + csv.get_csv()
            if log:
                log.write(line + '\n')
                log.flush()
            if csv.has_changed():
                print(line + '  ## ' + csv.get_change())
                sys.stdout.flush()
            elif flag_all or (flag_monitor and (count % 100 == 0)):
                print(line)
                sys.stdout.flush()
            csv.update()
            count += 1

            if flag_flash:
                if flag_erase:
                    flash_page_erase(s, pg)
                flash_prg(s, pg, off)
                pg = (pg + 1) & 0xff
                if pg == 0:
                    off = (off + 32) & 0xff
                    flag_erase = (off == 0)

            if flag_iram_err:
                iram_err_cnt = (iram_err_cnt + 1) & 3
                if iram_err_cnt == 0:
                    print("iram err at 0x{:x}".format(iram_err_addr))
                    s.cmd("write 0x80050 0x{:x}".format(iram_err_addr | 1))
                    iram_err_addr += 4
                    if iram_err_addr > 0xa00:
                        iram_err_addr = 0
        except CommError:
            print("# Comm error")
            csv.discard()

    s.close()

run()
