Solaris core analysis

root@sonada:/var/crash/sonada# savecore  -f /var/crash/sonada/vmdump.72 /var/crash/
savecore: System dump time: Fri Dec 12 12:22:49 2014

savecore: saving system crash dump in /var/crash//{unix,vmcore}.72
Constructing namelist /var/crash//unix.72
Constructing corefile /var/crash//vmcore.72
 0:22 100% done: 220581 of 220581 pages saved


root@sonada:/var/crash# mdb unix.72 vmcore.72
Loading modules: [ unix genunix specfs dtrace mac cpu.generic uppc pcplusmp scsi_vhci zfs ip hook neti sockfs arp usba uhci stmf stmf_sbd lofs random idm nfs sppp crypto ptm ipc ]
> ::status
debugging crash dump vmcore.72 (64-bit) from sonada
operating system: 5.11 oi_151a8 (i86pc)
image uuid: 2208e51c-870f-6f20-e9bd-b86a73b9806c
panic message: CONN_DEC_REF: connp(ffffff0374d34140) has ref = 0

dump content: kernel pages only


> ::msgbuf
MESSAGE
NOTICE: t4nex2: write-combine: using value 1
WARNING: t4nex2: Successfully configured using Firmware Configuration File version 0x14250014, computed checksum 0x565a5d42

pcplusmp: pciex1425,5401 (t4nex) instance 2 irq 0x1c vector 0x62 ioapic 0xff intin 0xff is bound to cpu 0
pcplusmp: pciex1425,5401 (t4nex) instance 2 irq 0x1d vector 0x63 ioapic 0xff intin 0xff is bound to cpu 1
pcplusmp: pciex1425,5401 (t4nex) instance 2 irq 0x1e vector 0x64 ioapic 0xff intin 0xff is bound to cpu 2
pcplusmp: pciex1425,5401 (t4nex) instance 2 irq 0x1f vector 0x65 ioapic 0xff intin 0xff is bound to cpu 3
pcplusmp: pciex1425,5401 (t4nex) instance 2 irq 0x21 vector 0x66 ioapic 0xff intin 0xff is bound to cpu 0
pcplusmp: pciex1425,5401 (t4nex) instance 2 irq 0x22 vector 0x67 ioapic 0xff intin 0xff is bound to cpu 1
pcplusmp: pciex1425,5401 (t4nex) instance 2 irq 0x23 vector 0x68 ioapic 0xff intin 0xff is bound to cpu 2
pcplusmp: pciex1425,5401 (t4nex) instance 2 irq 0x24 vector 0x69 ioapic 0xff intin 0xff is bound to cpu 3
pcplusmp: pciex1425,5401 (t4nex) instance 2 irq 0x25 vector 0x6a ioapic 0xff intin 0xff is bound to cpu 0
pcplusmp: pciex1425,5401 (t4nex) instance 2 irq 0x26 vector 0x6b ioapic 0xff intin 0xff is bound to cpu 1
pcplusmp: pciex1425,5401 (t4nex) instance 2 irq 0x27 vector 0x6c ioapic 0xff intin 0xff is bound to cpu 2
pcplusmp: pciex1425,5401 (t4nex) instance 2 irq 0x28 vector 0x6d ioapic 0xff intin 0xff is bound to cpu 3
pcplusmp: pciex1425,5401 (t4nex) instance 2 irq 0x29 vector 0x6e ioapic 0xff intin 0xff is bound to cpu 0
pcplusmp: pciex1425,5401 (t4nex) instance 2 irq 0x2a vector 0x6f ioapic 0xff intin 0xff is bound to cpu 1
PCIE-device: pci1425,0@0,4, t4nex2
PCI Express-device: pci1425,0@0,4, t4nex2
t4nex2 is /pci@0,0/pci8086,340c@5/pci1425,0@0,4
NOTICE: t4nex2: 2x10G (4 rxq, 4 txq, 2 ofld_rxq, 4 ofld_txq per port) 14 MSI-X interrupts.
NOTICE: cxgbe4: Multiple Rings Enabled
NOTICE: cxgbe4 registered
cxgbe4 is port 0 on t4nex2
cxgbe4 is /pci@0,0/pci8086,340c@5/pci1425,0@0,4/cxgbe@0
NOTICE: cxgbe5: Multiple Rings Enabled
NOTICE: cxgbe5 registered
cxgbe5 is port 1 on t4nex2
cxgbe5 is /pci@0,0/pci8086,340c@5/pci1425,0@0,4/cxgbe@1
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
pseudo-device: dtrace0
dtrace0 is /pseudo/dtrace@0
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
pcplusmp: asy (asy) instance 0 irq 0x4 vector 0xb0 ioapic 0x1 intin 0x4 is bound to cpu 2
ISA-device: asy0
asy0 is /pci@0,0/isa@1f/asy@1,3f8
pcplusmp: asy (asy) instance 1 irq 0x3 vector 0xb1 ioapic 0x1 intin 0x3 is bound to cpu 3
ISA-device: asy1
asy1 is /pci@0,0/isa@1f/asy@1,2f8
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
xsvc0 at root: space 0 offset 0
xsvc0 is /xsvc@0,0
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
NOTICE: IRQ19 is being shared by drivers with different interrupt levels.
This may result in reduced system performance.
pseudo-device: pool0
pool0 is /pseudo/pool@0
NOTICE: cxgbe4 unregistered
NOTICE: cxgbe5 unregistered
NOTICE: e1000g1 unregistered
pseudo-device: pm0
pm0 is /pseudo/pm@0
NOTICE: cxgbe4: Multiple Rings Enabled
NOTICE: cxgbe4 registered
cxgbe4 is port 0 on t4nex2
cxgbe4 is /pci@0,0/pci8086,340c@5/pci1425,0@0,4/cxgbe@0
pseudo-device: devinfo0
devinfo0 is /pseudo/devinfo@0
pcplusmp: pciex8086,10d3 (e1000g) instance 1 irq 0x1b vector 0x61 ioapic 0xff intin 0xff is bound to cpu 0
NOTICE: e1000g1 registered
NOTICE: cxgbe4 link up, 10000 Mbps, full duplex
NOTICE: cxgbe5: Multiple Rings Enabled
NOTICE: cxgbe5 registered
cxgbe5 is port 1 on t4nex2
cxgbe5 is /pci@0,0/pci8086,340c@5/pci1425,0@0,4/cxgbe@1
NOTICE: cxgbe5 link up, 10000 Mbps, full duplex

panic[cpu1]/thread=ffffff0013806c40:
CONN_DEC_REF: connp(ffffff0374d34140) has ref = 0


ffffff0013806a10 toe:do_abort_rpl_squeue+36b ()
ffffff0013806a60 t4nex:handle_fw_rpl+87 ()
ffffff0013806b10 t4nex:t4_ring_rx+3af ()
ffffff0013806b50 t4nex:t4_intr_rx_work+8b ()
ffffff0013806b90 t4nex:t4_intr+4f ()
ffffff0013806be0 unix:av_dispatch_autovect+7c ()
ffffff0013806c20 unix:dispatch_hardint+33 ()
ffffff00137d6a00 unix:switch_sp_and_call+13 ()
ffffff00137d6a50 unix:do_interrupt+b8 ()
ffffff00137d6a60 unix:cmnint+ba ()
ffffff00137d6bc0 unix:acpi_cpu_cstate+2ae ()
ffffff00137d6bf0 unix:cpu_acpi_idle+82 ()
ffffff00137d6c00 unix:cpu_idle_adaptive+19 ()
ffffff00137d6c20 unix:idle+114 ()
ffffff00137d6c30 unix:thread_start+8 ()

syncing file systems...
 done
dumping to /dev/zvol/dsk/rpool/dump, offset 65536, content: kernel