Retry failed core test a few times for 90 days until we can fix inconsistent tests (#1218)

ci/test.sh

@@ -34,6 +34,8 @@ xvfb_run_() {
 }
 
 run_tests() {
+    local tries try
+
     # when busybox pretends to be bash it needs different args
     #   for the timeout builtin
     if [[ "${BUSYBOX_BASH}" -eq 1 ]]; then
@@ -42,8 +44,26 @@ run_tests() {
         TIMEOUT_TIME_ARG=""
     fi
 
-    ${TIMEOUT_CMD} ${TIMEOUT_TIME_ARG} ${TIMEOUT_SEC-${TIMEOUT_DEFAULT}} ./core_test
-    core_test_res=${?}
+    if [ "$(date +%s)" -lt 1545350400 ]; then
+        tries=(1 2 3 4 5 6 7 8 9)
+    else
+        tries=()
+    fi
+
+    for try in _initial_ "${tries[@]}"; do
+        if [ "${try}" != '_initial_' ]; then
+            echo "core_test failed: ${core_test_res}, retrying (try=${try})"
+
+            # Wait a while for sockets to be all cleaned up by the kernel
+            sleep $[10 + (${RANDOM} % 30)]
+        fi
+
+        ${TIMEOUT_CMD} ${TIMEOUT_TIME_ARG} ${TIMEOUT_SEC-${TIMEOUT_DEFAULT}} ./core_test
+        core_test_res=${?}
+        if [ "${core_test_res}" = '0' ]; then
+            break
+        fi
+    done
 
     xvfb_run_ ./qt_test
     qt_test_res=${?}