Merge branch 'hashcat:master' into metal_prepare_kernelIncludes_v2

docs/changes.txt

@@ -66,6 +66,7 @@
 - OpenCL Backend: moved functions to ext_OpenCL.c and includes to ext_OpenCL.h
 - HIP Backend: moved functions to ext_hip.c/ext_hiprtc.c and includes to ext_hip.h/ext_hiprtc.h
 - CUDA Backend: moved functions to ext_cuda.c/ext_nvrtc.c and includes to ext_cuda.h/ext_nvrtc.h
+- Autotune: Added error handling. By default skipping device on error, with --force using accel/loops/threads min values instead
 - Makefile: updated MACOSX_DEPLOYMENT_TARGET to 10.15 and removed OpenCL framework from LFLAGS_NATIVE on MacOS
 - Status code: updated negative status code, usefull in Unit tests engine (test.sh)
 - Unit tests: added -r (--runtime) option

docs/status_codes.txt

@@ -1,14 +1,15 @@
 status codes on exit:
 =====================
 
--8 = mixed backend errors (combo of -3, -4, -5, -6, -7 errors type)
--7 = backend error: Invalid module_extra_buffer_size
--6 = backend error: Too many compute units to keep minimum kernel accel limit
--5 = backend error: main kernel build error
--4 = backend error: memory hit
--3 = backend error: skipping hash-type due to module_unstable_warning settings
--2 = gpu-watchdog alarm
--1 = error
+-10 = autotune failure
+ -8 = mixed backend errors (combo of -3, -4, -5, -6, -7 errors type)
+ -7 = backend error: Invalid module_extra_buffer_size
+ -6 = backend error: Too many compute units to keep minimum kernel accel limit
+ -5 = backend error: main kernel build error
+ -4 = backend error: memory hit
+ -3 = backend error: skipping hash-type due to module_unstable_warning settings
+ -2 = gpu-watchdog alarm
+ -1 = error
   0 = OK/cracked
   1 = exhausted
   2 = aborted

include/types.h

@@ -197,6 +197,13 @@ typedef enum st_status_rc
 
 } st_status_t;
 
+typedef enum at_status_rc
+{
+  AT_STATUS_PASSED        = 0,
+  AT_STATUS_FAILED        = 1,
+
+} at_status_t;
+
 typedef enum status_rc
 {
   STATUS_INIT               = 0,
@@ -1127,7 +1134,11 @@ typedef struct hc_device_param
 
   u32     kernel_preferred_wgs_multiple;
 
-  st_status_t st_status;
+  st_status_t st_status;        // selftest status
+
+  at_status_t at_status;        // autotune status
+
+  int     at_rc;                // autotune rc
 
   int     vector_width;
 

src/autotune.c

@@ -120,6 +120,20 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param
   const u32 kernel_threads_min = device_param->kernel_threads_min;
   const u32 kernel_threads_max = device_param->kernel_threads_max;
 
+  // stores the minimum values
+  // they could be used if the autotune fails and user specify --force
+
+  if (user_options->force == true)
+  {
+    device_param->kernel_accel   = kernel_accel_min;
+    device_param->kernel_loops   = kernel_loops_min;
+    device_param->kernel_threads = kernel_threads_min;
+    device_param->hardware_power = ((hashconfig->opts_type & OPTS_TYPE_MP_MULTI_DISABLE) ? 1 : device_param->device_processors) * kernel_threads_min;
+    device_param->kernel_power   = device_param->hardware_power * kernel_accel_min;
+  }
+
+  // start engine
+
   u32 kernel_accel = kernel_accel_min;
   u32 kernel_loops = kernel_loops_min;
 
@@ -212,6 +226,8 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param
       }
     }
 
+    device_param->at_rc = -2;
+
     if (device_param->is_cuda == true)
     {
       if (run_cuda_kernel_atinit (hashcat_ctx, device_param, device_param->cuda_d_pws_buf, kernel_power_max) == -1) return -1;
@@ -236,6 +252,8 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param
       {
         if (straight_ctx->kernel_rules_cnt > 1)
         {
+          device_param->at_rc = -3;
+
           if (device_param->is_cuda == true)
           {
             if (hc_cuMemcpyDtoDAsync (hashcat_ctx, device_param->cuda_d_rules_c, device_param->cuda_d_rules, MIN (kernel_loops_max, KERNEL_RULES) * sizeof (kernel_rule_t), device_param->cuda_stream) == -1) return -1;
@@ -290,6 +308,8 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param
       {
         event_log_error (hashcat_ctx, "Kernel minimum runtime larger than default TDR");
 
+        device_param->at_rc = -4;
+
         return -1;
       }
 
@@ -437,44 +457,36 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param
   // reset them fake words
   // reset other buffers in case autotune cracked something
 
+  device_param->at_rc = -5;
+
   if (device_param->is_cuda == true)
   {
     if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_pws_buf, device_param->size_pws) == -1) return -1;
-
     if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_plain_bufs, device_param->size_plains) == -1) return -1;
-
     if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_digests_shown, device_param->size_shown) == -1) return -1;
-
     if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_result, device_param->size_results) == -1) return -1;
-
     if (run_cuda_kernel_bzero (hashcat_ctx, device_param, device_param->cuda_d_tmps, device_param->size_tmps) == -1) return -1;
   }
 
   if (device_param->is_hip == true)
   {
     if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_pws_buf, device_param->size_pws) == -1) return -1;
-
     if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_plain_bufs, device_param->size_plains) == -1) return -1;
-
     if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_digests_shown, device_param->size_shown) == -1) return -1;
-
     if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_result, device_param->size_results) == -1) return -1;
-
     if (run_hip_kernel_bzero (hashcat_ctx, device_param, device_param->hip_d_tmps, device_param->size_tmps) == -1) return -1;
   }
 
   if (device_param->is_opencl == true)
   {
     if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_pws_buf, device_param->size_pws) == -1) return -1;
-
     if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_plain_bufs, device_param->size_plains) == -1) return -1;
-
     if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_digests_shown, device_param->size_shown) == -1) return -1;
-
     if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_result, device_param->size_results) == -1) return -1;
-
     if (run_opencl_kernel_bzero (hashcat_ctx, device_param, device_param->opencl_d_tmps, device_param->size_tmps) == -1) return -1;
 
+    device_param->at_rc = -6;
+
     if (hc_clFlush (hashcat_ctx, device_param->opencl_command_queue) == -1) return -1;
   }
 
@@ -483,7 +495,6 @@ static int autotune (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param
   device_param->exec_pos = 0;
 
   memset (device_param->exec_msec,          0,          EXEC_CACHE * sizeof (double));
-
   memset (device_param->exec_us_prev1,      0, EXPECTED_ITERATIONS * sizeof (double));
   memset (device_param->exec_us_prev2,      0, EXPECTED_ITERATIONS * sizeof (double));
   memset (device_param->exec_us_prev3,      0, EXPECTED_ITERATIONS * sizeof (double));
@@ -517,7 +528,6 @@ HC_API_CALL void *thread_autotune (void *p)
   thread_param_t *thread_param = (thread_param_t *) p;
 
   hashcat_ctx_t *hashcat_ctx = thread_param->hashcat_ctx;
-
   backend_ctx_t *backend_ctx = hashcat_ctx->backend_ctx;
 
   if (backend_ctx->enabled == false) return NULL;
@@ -527,6 +537,11 @@ HC_API_CALL void *thread_autotune (void *p)
   if (device_param->skipped == true) return NULL;
   if (device_param->skipped_warning == true) return NULL;
 
+  // init autotunes status and rc
+
+  device_param->at_status = AT_STATUS_FAILED;
+  device_param->at_rc = -1; // generic error
+
   if (device_param->is_cuda == true)
   {
     if (hc_cuCtxPushCurrent (hashcat_ctx, device_param->cuda_context) == -1) return NULL;
@@ -537,11 +552,12 @@ HC_API_CALL void *thread_autotune (void *p)
     if (hc_hipCtxPushCurrent (hashcat_ctx, device_param->hip_context) == -1) return NULL;
   }
 
-  const int rc_autotune = autotune (hashcat_ctx, device_param);
+  // check for autotune failure
 
-  if (rc_autotune == -1)
+  if (autotune (hashcat_ctx, device_param) == 0)
   {
-    // we should do something here, tell hashcat main that autotune failed to abort
+    device_param->at_status = AT_STATUS_PASSED;
+    device_param->at_rc = 0;
   }
 
   if (device_param->is_cuda == true)

src/hashcat.c

@@ -211,6 +211,66 @@ static int inner2_loop (hashcat_ctx_t *hashcat_ctx)
 
   hc_thread_wait (backend_ctx->backend_devices_cnt, c_threads);
 
+  // check for any autotune failures
+  // by default, skipping device on error
+  // using --force, accel/loops/threads min values are used instead of skipping
+
+  int at_err = 0;
+
+  for (int backend_devices_idx = 0; backend_devices_idx < backend_ctx->backend_devices_cnt; backend_devices_idx++)
+  {
+    if (backend_ctx->enabled == false) continue;
+
+    hc_device_param_t *device_param = backend_ctx->devices_param + backend_devices_idx;
+
+    if (device_param->skipped == true) continue;
+
+    if (device_param->skipped_warning == true) continue;
+
+    if (device_param->at_status == AT_STATUS_FAILED)
+    {
+      at_err++;
+
+      if (user_options->force == false)
+      {
+        event_log_warning (hashcat_ctx, "* Device #%u: skipped, due to kernel autotune failure (%d).", device_param->device_id + 1, device_param->at_rc);
+
+        device_param->skipped = true;
+
+        // update counters
+
+        if (device_param->is_hip == true)    backend_ctx->hip_devices_active--;
+        if (device_param->is_cuda == true)   backend_ctx->cuda_devices_active--;
+        if (device_param->is_opencl == true) backend_ctx->opencl_devices_active--;
+
+        backend_ctx->backend_devices_active--;
+      }
+      else
+      {
+        event_log_warning (hashcat_ctx, "* Device #%u: detected kernel autotune failure (%d), min values will be used", device_param->device_id + 1, device_param->at_rc);
+      }
+    }
+  }
+
+  if (at_err > 0)
+  {
+    event_log_warning (hashcat_ctx, NULL);
+
+    if (user_options->force == false)
+    {
+      // if all enabled devices fail, abort session
+      if (backend_ctx->backend_devices_active <= 0)
+      {
+        event_log_error (hashcat_ctx, "Aborting session due to kernel autotune failures, for all active devices.");
+
+        event_log_warning (hashcat_ctx, "You can use --force to override this, but do not report related errors.");
+        event_log_warning (hashcat_ctx, NULL);
+
+        return -10;
+      }
+    }
+  }
+
   EVENT (EVENT_AUTOTUNE_FINISHED);
 
   /**

tools/test.sh

@@ -403,6 +403,12 @@ function status()
 
   if [ "${RET}" -ne 0 ]; then
     case ${RET} in
+      246)
+        echo "autotune failure, cmdline : ${CMD}" >> "${OUTD}/logfull.txt" 2>> "${OUTD}/logfull.txt"
+
+        e_rs=$((e_rs + 1))
+        ;;
+
       248)
         echo "skipped by runtime (mixed backend errors detected), cmdline : ${CMD}" >> "${OUTD}/logfull.txt" 2>> "${OUTD}/logfull.txt"