samsung: snow: bitfix: Panic if too many bits fixed in 1 chunk

If we see too much corruption in a single chunk then that's a sign
that we're in a state that we can't recover from (either a bitfix
internal error or a corruption of some memory that can't be checked).
Panic in that case.

BUG=chrome-os-partner:15655
TEST=suspend_stress_test
TEST=In code change MAX_BYTES_TO_FIX to 3 and then re-run test.
See a reboot and see the panic in /var/spool/crash:
  <7>[  163.818802] s3c_pm_check: Restore CRC error at 71734000 (6da6143e vs 1d894763)
  <6>[  163.818802] bitfix_recover_chunk: Attempting recovery at 71734000
  <6>[  163.818802] ...fixed 0x71735644 from 0xbadc0de1 to 0x00000000 (4 bytes)
  <5>[  163.818802] ------------[ cut here ]------------
  <2>[  163.818802] kernel BUG at .../arch/arm/mach-exynos/bitfix-snow.c:504!
  <0>[  163.818802] Internal error: Oops - BUG: 0 [#1] SMP ARM
  <5>[  163.818802] Modules linked in: i2c_dev uinput isl29018(C) industrialio(C) sbs_battery rfcomm btmrvl_sdio btmrvl bluetooth fuse nf_conntrack_ipv6 nf_defrag_ipv6 ip6table_filter xt_mark ip6_tables mwifiex_sdio mwifiex cfg80211 spidev uvcvideo videobuf2_vmalloc asix smsc95xx usbnet joydev
  <5>[  163.818802] CPU: 0    Tainted: G         C    (3.4.0 #4)
  <5>[  163.818802] PC is at bitfix_recover_chunk+0x220/0x244
  <5>[  163.818802] LR is at bitfix_recover_chunk+0x20c/0x244
  ...
  <5>[  163.818802] [<80024f5c>] (bitfix_recover_chunk+0x220/0x244) from [<8002786c>] (s3c_pm_runcheck+0x158/0x2a0)
  ...

Change-Id: Iaad53d821231f419cb1e44a9e1ef3c8ec36973d2
Signed-off-by: Doug Anderson <dianders@chromium.org>
Reviewed-on: https://gerrit.chromium.org/gerrit/37600
Reviewed-by: Jon Kliegman <kliegs@chromium.org>

diff --git a/arch/arm/mach-exynos/bitfix-snow.c b/arch/arm/mach-exynos/bitfix-snow.c
index 6792912..14e54fe 100644 (file)
--- a/arch/arm/mach-exynos/bitfix-snow.c
+++ b/arch/arm/mach-exynos/bitfix-snow.c
@@ -219,6 +219,20 @@
  */
 #define XOR_CU_NUM             (CU_COUNT / 2)
 
+/*
+ * Corruption should be limited to a few hundred bytes in general and that's
+ * across all chunks.  If we see a large number of bytes corrupted in a single
+ * chunk then that's a sign that data we depend on has been corrupted and we
+ * can not trust our state.
+ *
+ * We'll be conservative and allow up to 1/16 bytes in a chunk before we panic.
+ *
+ * We have seen a case like this in testing where 3 pages were nearly
+ * completely 'fixed' and the fix matched the out-of-band checksum.  The best
+ * guess is that page tables were corrupted and confusing bitfix.  Reboot is
+ * best in this case.
+ */
+#define MAX_BYTES_TO_FIX       (CHUNK_SIZE / 16)
 
 /* Functionality is automatically enabled/disabled at boot time based on need */
 static bool bitfix_enabled;
@@ -310,22 +324,48 @@ static void bitfix_xor_page(phys_addr_t page_addr, u32 dest_cu)
 }
 
 /**
- * Print out which bits were fixed in a page.
+ * Return how many bytes are different in two words.
+ *
+ * @x: First word
+ * @y: Second word
+ * @return: Number of bytes different (0 - 4).
+ */
+static int compare_words(u32 x, u32 y)
+{
+       int diff = 0;
+       while (x != y) {
+               if ((x & 0xff) != (y & 0xff))
+                       diff++;
+               x = x >> 8;
+               y = y >> 8;
+       }
+
+       return diff;
+}
+
+/**
+ * Analyze (and print out) which bits were fixed in a page.
  *
  * @addr: The address of the page that was recovered.
  * @orig: The old (corrupted) data.
  * @recovered: The new data.
+ * @return: Number of bytes fixed
  */
 
-static void bitfix_compare(phys_addr_t addr, const u32 *orig,
+static int bitfix_compare(phys_addr_t addr, const u32 *orig,
                           const u32 *recovered)
 {
        int i;
+       int bytes_fixed = 0;
 
        for (i = 0; i < PAGE_SIZE/4; i++)
-               if (orig[i] != recovered[i])
+               if (orig[i] != recovered[i]) {
+                       bytes_fixed += compare_words(orig[i], recovered[i]);
                        pr_info("...fixed 0x%08x from 0x%08x to 0x%08x\n",
                                addr + (i * 4), orig[i], recovered[i]);
+               }
+
+       return bytes_fixed;
 }
 
 /**
@@ -430,6 +470,7 @@ void bitfix_recover_chunk(phys_addr_t failed_addr,
 {
        const phys_addr_t bad_chunk_addr = failed_addr & ~(CHUNK_MASK);
        int pgnum;
+       int bytes_fixed = 0;
 
        if (!bitfix_enabled)
                return;
@@ -453,10 +494,12 @@ void bitfix_recover_chunk(phys_addr_t failed_addr,
                u32 *recover_page = recover_chunk + offset / sizeof(u32);
 
                virt = kmap_atomic(phys_to_page(addr));
-               bitfix_compare(addr, virt, recover_page);
+               bytes_fixed += bitfix_compare(addr, virt, recover_page);
                memcpy(virt, recover_page, PAGE_SIZE);
                kunmap_atomic(virt);
        }
+
+       BUG_ON(bytes_fixed > MAX_BYTES_TO_FIX);
 }
 
 /**