WD Sentinel DS5100 (NAS) boot issues

[DeanWy]

Hi all,

I have a Western Digital Sentinel DS5100 NAS system which has stopped working. It has dual BIOS. I bought the system second hand with an already broken primary BIOS (it rebooted multiple times until second BIOS was used).

It worked until recently. The system was idle in Windows Server 2012 and a BSOD occurred, I think it was 'critical process died'. Thereafter it didn't boot, it just sat there with no video output and fan speed at 100% (there's a proprietary Windows driver by WD which controls the fan, outside of that it always runs at 100%).

There is a BMC onboard (ASPEED AST2300) to which I can connect through a serial connection. There's a bmcport80 driver there developed by WD which does the BIOS checks and switches between primary and secondary if the primary fails (it seems to do that by using GPIO, see decompiled code below).
In the BMC logs I noticed that mounting the flash fails, I assume the BIOS flash:

mount: mounting dev/mtdblock0 on /mnt/flash0 failed: No such device or address
mount: mounting dev/mtdblock1 on /mnt/flash1 failed: No such device or address

I thought this indicated that maybe the BIOS flash chips both died. So I desoldered the BIOS chips and tried reading the chips. One of them was indeed broken (the primary BIOS), the other one worked just fine so I was wrong... I dumped the BIOS ROM nonetheless and reflashed two new chips and soldered them back.

They are the ones on the left and bottom.



Now however there's even less output. The fan doesn't spin at 100% anymore but like 5-10%.

Previously the logs said:

Scheduled delayed BIOS check - SLP_S3#=high, PCH_BIOS_OK=low
... snip
# timer event 11
CMD_DELAYED_CHECK
Switched to Backup BIOS - SLP_S3#=high, PCH_BIOS_OK=low
PWRBTSW
timer event 8
CMD_REBOOT_HOST
timer event 9
PWRBTSW
CMD_REBOOT_HOST_SM1
timer event 0
CMD_RELEASE_RST
SLP_S3# -> HI count=1
MEM_EVENT[3:0]=0xf
tasklet count=1
timer event# 10 start: 90 sec
timer event 10
CMD_IS_BIOS_OK?
In Backup BIOS already

But now it says it doesn't schedule the delayed BIOS check, and it says:

SLP_S3# -> HI count=1
MEM_EVENT[3:0]=0xf
tasklet count=1
timer event# 10 start: 90 sec
SLP_S3# -> HI count=2
MEM_EVENT[3:0]=0xf
tasklet count=2
timer event# 10 start: 90 sec
SLP_S3# -> HI count=3
MEM_EVENT[3:0]=0xf
tasklet count=3
timer event# 10 start: 90 sec
SLP_S3# -> HI count=4
MEM_EVENT[3:0]=0xf
tasklet count=4
timer event# 10 start: 90 sec
SLP_S3# -> HI count=5
MEM_EVENT[3:0]=0xf
tasklet count=5
timer event# 10 start: 90 sec
SLP_S3# -> HI count=6
MEM_EVENT[3:0]=0xf
tasklet count=6
timer event# 10 start: 90 sec
SLP_S3# -> HI count=7
Switched to Backup BIOS - count exceeded
MEM_EVENT[3:0]=0xf
SLP_S3# -> HI count=1
MEM_EVENT[3:0]=0xf
tasklet count=1
timer event# 10 start: 90 sec
SLP_S3# -> HI count=2
MEM_EVENT[3:0]=0xf
tasklet count=2
timer event# 10 start: 90 sec
SLP_S3# -> HI count=3
MEM_EVENT[3:0]=0xf
tasklet count=3
timer event# 10 start: 90 sec
SLP_S3# -> HI count=4
MEM_EVENT[3:0]=0xf
tasklet count=4
timer event# 10 start: 90 sec
SLP_S3# -> HI count=5
MEM_EVENT[3:0]=0xf
tasklet count=5
timer event# 10 start: 90 sec
timer event# 10 start: 90 sec
SLP_S3# -> HI count=7
Main & Backup failed - Hold backplane in reset.
MEM_EVENT[3:0]=0xf
SLP_S3# -> HI count=8
Main & Backup failed - Hold backplane in reset.​

I checked the condition for the delayed BIOS check but I have no clue what this checks:

 if (((*(uint *)((int)gpio_base_addr + 0x70) & 0x2000000) != 0) &&  ((*(uint *)((int)gpio_base_addr + 0x78) & 0x18) == 0)) {
     slp_s3_counter = 0;
    printk(" <6>Scheduled delayed BIOS check - SLP_S3#=high, PCH_BIOS_OK=low\n");
     initWorkTimer(0xb);
    return 0;
 }

I attached the full logs below, and also the loaded kernel modules.

lsmod
mailbox 8732 0 - Live 0xbf00b000
aspeed_i2c 4680 0 - Live 0xbf007000
adc 3748 0 - Live 0xbf004000
bmcport80 7980 0 - Live 0xbf000000

Weirdly enough no mtd devices are found.

cat /proc/mtd
dev: size erasesize name

Parts of bmcport80 decompiled code using Ghidra:

undefined4 gpio_interrupt(void)
{
void *pvVar1;
uint uVar2;
uint uVar3;

uVar3 = *(uint *)((int)gpio_base_addr + 0x38);
if ((uVar3 & 0x40) != 0) {
*(undefined4 *)((int)gpio_base_addr + 0x38) = 0x40;
printk("<6>PWRBTSW\n");
}
if ((uVar3 & 1) != 0) {
*(undefined4 *)((int)gpio_base_addr + 0x38) = 1;
printk("<6>RECOVER\n");
}
uVar3 = *(uint *)((int)gpio_base_addr + 0x18);
if ((uVar3 & 0xc0000000) != 0) {
*(undefined4 *)((int)gpio_base_addr + 0x18) = 0xc0000000;
printk("<6>ETH_WAKE[1:0]=0x%lx\n",uVar3 >> 0x1e);
WakeOnLan();
}
pvVar1 = gpio_base_addr;
if ((uVar3 & 0x3c000000) != 0) {
*(undefined4 *)((int)gpio_base_addr + 0x18) = 0x3c000000;
uVar3 = *(uint *)((int)pvVar1 + 0x70);
if (((uVar3 & 0x2000000) != 0) && ((*(uint *)((int)pvVar1 + 0x78) & 8) != 0)) {
printk("<6>MEM_EVENT[3:0]=0x%lx\n",uVar3 >> 0x1a & 0xf);
}
}
if ((uVar3 & 0x200) != 0) {
*(undefined4 *)((int)gpio_base_addr + 0x18) = 0x200;
printk("<6>ASF_SMBALERT#\n");
}
if ((uVar3 & 0x800) != 0) {
*(undefined4 *)((int)gpio_base_addr + 0x18) = 0x800;
printk("<6>LINK_ALT#\n");
}
if ((*(uint *)((int)gpio_base_addr + 0xa8) & 0x2000000) != 0) {
slp_s3_counter = slp_s3_counter + 1;
printk("<6>SLP_S3# -> HI count=%d\n",slp_s3_counter);
if (slp_s3_counter < 7) {
uVar2 = my_tasklet.state | 1;
uVar3 = my_tasklet.state & 1;
my_tasklet.state = uVar2;
if (uVar3 == 0) {
__tasklet_schedule(&my_tasklet);
}
}
else if ((*(uint *)((int)gpio_base_addr + 0x78) & 0x10) == 0) {
*(uint *)((int)gpio_base_addr + 0x78) = *(uint *)((int)gpio_base_addr + 0x78) | 0x10;
slp_s3_counter = 0;
printk("<6>Switched to Backup BIOS - count exceeded\n");
}
else {
printk("<6>Main & Backup failed - Hold backplane in reset.\n");
/* WARNING: Load size is inaccurate */
*(uint *)gpio_base_addr = *gpio_base_addr & 0xfeffffff;
slp_s3_counter = 7;
}
*(undefined4 *)((int)gpio_base_addr + 0xa8) = 0x2000000;
}
return 1;
}
}

void timer_function(undefined4 param_1)

{
char *pcVar1;

printk("<6>timer event %lu\n",param_1);
switch(param_1) {
case 0:
ich_pwrbt(1);
if (my_timer.entry.next != (list_head *)0x0) {
del_timer();
}
pcVar1 = "<6>CMD_RELEASE_RST\n";
break;
default:
printk("<3>bmcport80 timer command error %lu\n",param_1);
return;
case 8:
ich_pwrbt(1);
my_timer.expires = jiffies + 0x32;
my_timer.data = 9;
mod_timer(&my_timer,my_timer.expires);
pcVar1 = "<6>CMD_REBOOT_HOST\n";
break;
case 9:
ich_pwrbt(0);
my_timer.expires = jiffies + 0x32;
my_timer.data = 0;
mod_timer(&my_timer,my_timer.expires);
pcVar1 = "<6>CMD_REBOOT_HOST_SM1\n";
break;
case 10:
if (my_timer.entry.next != (list_head *)0x0) {
del_timer();
}
if ((*(uint *)((int)gpio_base_addr + 0x70) & 0x2000000) == 0) {
return;
}
printk("<6>CMD_IS_BIOS_OK?\n");
if ((*(uint *)((int)gpio_base_addr + 0x78) & 0x10) == 0) {
if ((*(uint *)((int)gpio_base_addr + 0x78) & 8) != 0) {
printk("<6>Main BIOS OK!\n");
slp_s3_counter = 0;
return;
}
pcVar1 = "<6>Switched to Backup BIOS - Main not OK\n";
*(uint *)((int)gpio_base_addr + 0x78) = *(uint *)((int)gpio_base_addr + 0x78) | 0x10;
LAB_00010560:
slp_s3_counter = 0;
printk(pcVar1);
ich_pwrbt(0);
my_timer.expires = jiffies + 500;
my_timer.data = 8;
mod_timer(&my_timer,my_timer.expires);
return;
}
pcVar1 = "<6>In Backup BIOS already\n";
break;
case 0xb:
if (my_timer.entry.next != (list_head *)0x0) {
del_timer();
}
printk("<6>CMD_DELAYED_CHECK\n");
if ((*(uint *)((int)gpio_base_addr + 0x70) & 0x2000000) != 0) {
if ((*(uint *)((int)gpio_base_addr + 0x78) & 0x18) != 0) {
return;
}
pcVar1 = "<6>Switched to Backup BIOS - SLP_S3#=high, PCH_BIOS_OK=low\n";
*(uint *)((int)gpio_base_addr + 0x78) = *(uint *)((int)gpio_base_addr + 0x78) | 0x10;
goto LAB_00010560;
}
pcVar1 = "<6>event ignored - SLP_S3=low CPU down.\n";
}
printk(pcVar1);
return;
}

Any advice is highly appreciated! There is so little documentation to be found about this device and it makes this very hard.

Thanks!

Dean

[DeanWy]

After looking at the code again I guess the first GPIO address (0x70 & 0x2000000 != 0) is SLP_S3 and the second one (0x78 & 0x18 == 0) is PCH_BIOS_OK as mentioned in the log message.

if (((*(uint *)((int)gpio_base_addr + 0x70) & 0x2000000) != 0) && ((*(uint *)((int)gpio_base_addr + 0x78) & 0x18) == 0)) {
...
printk(" <6>Scheduled delayed BIOS check - SLP_S3#=high, PCH_BIOS_OK=low\n");
...

As further logging indicates that SLP_S3 is indeed high (ie SLP_S3# -> HI count=3), we could assume that PCH_BIOS_OK is high as this message is not logged, right? Would the high signal mean that the BIOS is in fact OK?

I tested the pins (5-8) of the Winbond chips and both BIOS chips show current on those pins.