PLL1 was misconfigured to run at 408MHz (way out of spec) instead of 204MHz. Corrected this by using DIRECT=1 instead of FBSEL=1.

firmware/common/hackrf_core.c

@@ -252,7 +252,7 @@ void cpu_clock_init(void)
             | CGU_PLL1_CTRL_PSEL(0)
             | CGU_PLL1_CTRL_NSEL(0)
 			| CGU_PLL1_CTRL_MSEL(16)
-			| CGU_PLL1_CTRL_FBSEL;
+			| CGU_PLL1_CTRL_DIRECT;
 
 	/* wait until stable */
 	while (!(CGU_PLL1_STAT & CGU_PLL1_STAT_LOCK));