Task 5461715

Name RF_SAVE_ALL_OUT_NOJRAN_IGNORE_THE_REST_validation_env_f_pred_71_16902_4_1
Workunit 4847002
Created 14 Jun 2024, 1:46:03 UTC
Sent 14 Jun 2024, 6:13:10 UTC
Report deadline 15 Jun 2024, 6:13:10 UTC
Received 14 Jun 2024, 6:34:28 UTC
Server state Over
Outcome Computation error
Client state Compute error
Exit status 12 (0x0000000C) Unknown error code
Computer ID 49778
Run time 34 sec
CPU time
Validate state Invalid
Credit 0.00
Device peak FLOPS 1.94 GFLOPS
Application version Generalized biomolecular modeling and design with RoseTTAFold All-Atom v0.02 (nvidia_alpha)
windows_x86_64
Peak working set size 1,858.57 MB
Peak swap size 8,557.70 MB
Peak disk usage 2.09 MB

Stderr output

<core_client_version>7.24.1</core_client_version>
<![CDATA[
<message>
 - exit code 12 (0xc)</message>
<stderr_txt>
Traceback (most recent call last):
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\cv2\rf2aa\predict.py", line 708, in <module>
    pred.predict(out_name+f'_{n}', 
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\cv2\rf2aa\predict.py", line 551, in predict
    logit_s, logit_aa_s, logit_pae, logit_pde, p_bind, pred_crds, alpha, pred_allatom, pred_lddt_binned,                msa_prev, pair_prev, state_prev = self.model(
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\ev0\lib\site-packages\torch\nn\modules\module.py", line 1051, in _call_impl
    return forward_call(*input, **kwargs)
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\cv2\rf2aa\RoseTTAFoldModel.py", line 358, in forward
    msa, pair, xyz, alpha_s, xyz_allatom, state, symmsub = self.simulator(
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\ev0\lib\site-packages\torch\nn\modules\module.py", line 1051, in _call_impl
    return forward_call(*input, **kwargs)
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\cv2\rf2aa\Track_module.py", line 1084, in forward
    msa_full, pair, xyz, state, alpha, symmsub = self.extra_block[i_m](msa_full, pair,
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\ev0\lib\site-packages\torch\nn\modules\module.py", line 1051, in _call_impl
    return forward_call(*input, **kwargs)
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\cv2\rf2aa\Track_module.py", line 929, in forward
    xyz, state, alpha = self.str2str(
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\ev0\lib\site-packages\torch\nn\modules\module.py", line 1051, in _call_impl
    return forward_call(*input, **kwargs)
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\ev0\lib\site-packages\torch\cuda\amp\autocast_mode.py", line 141, in decorate_autocast
    return func(*args, **kwargs)
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\cv2\rf2aa\Track_module.py", line 503, in forward
    shift = self.se3(G, node.reshape(B*L, -1, 1), l1_feats, edge_feats)
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\ev0\lib\site-packages\torch\nn\modules\module.py", line 1051, in _call_impl
    return forward_call(*input, **kwargs)
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\cv2\rf2aa\SE3_network.py", line 96, in forward
    return self.se3(G, node_features, edge_features)
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\ev0\lib\site-packages\torch\nn\modules\module.py", line 1051, in _call_impl
    return forward_call(*input, **kwargs)
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\cv2\rf2aa/SE3Transformer\se3_transformer\model\transformer.py", line 163, in forward
    basis = basis or get_basis(graph.edata['rel_pos'], max_degree=self.max_degree, compute_gradients=False,
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\cv2\rf2aa/SE3Transformer\se3_transformer\model\basis.py", line 167, in get_basis
    spherical_harmonics = get_spherical_harmonics(relative_pos, max_degree)
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\cv2\rf2aa/SE3Transformer\se3_transformer\model\basis.py", line 58, in get_spherical_harmonics
    sh = o3.spherical_harmonics(all_degrees, relative_pos, normalize=True)
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\ev0\lib\site-packages\e3nn\o3\_spherical_harmonics.py", line 180, in spherical_harmonics
    return sh(x)
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\ev0\lib\site-packages\torch\nn\modules\module.py", line 1051, in _call_impl
    return forward_call(*input, **kwargs)
  File "E:\Data\BOINC\projects\ralph.bakerlab.org\ev0\lib\site-packages\e3nn\o3\_spherical_harmonics.py", line 82, in forward
    sh = _spherical_harmonics(self._lmax, x[..., 0], x[..., 1], x[..., 2])
RuntimeError: nvrtc: error: invalid value for --gpu-architecture (-arch)

nvrtc compilation failed: 

#define NAN __int_as_float(0x7fffffff)
#define POS_INFINITY __int_as_float(0x7f800000)
#define NEG_INFINITY __int_as_float(0xff800000)


template<typename T>
__device__ T maximum(T a, T b) {
  return isnan(a) ? a : (a > b ? a : b);
}

template<typename T>
__device__ T minimum(T a, T b) {
  return isnan(a) ? a : (a < b ? a : b);
}

extern "C" __global__
void fused_pow_pow_pow_su_3082509318163616828(float* tz_1, float* ty_1, float* tx_1, float* aten_mul, float* aten_mul_1, float* aten_mul_2, float* aten_sub, float* aten_add, float* aten_mul_3, float* aten_pow) {
{
  if (512 * blockIdx.x + threadIdx.x<64768 ? 1 : 0) {
    float ty_1_1 = __ldg(ty_1 + 3 * (512 * blockIdx.x + threadIdx.x));
    aten_pow[512 * blockIdx.x + threadIdx.x] = ty_1_1 * ty_1_1;
    float tz_1_1 = __ldg(tz_1 + 3 * (512 * blockIdx.x + threadIdx.x));
    float tx_1_1 = __ldg(tx_1 + 3 * (512 * blockIdx.x + threadIdx.x));
    aten_mul_3[512 * blockIdx.x + threadIdx.x] = (float)((double)(tz_1_1 * tz_1_1 - tx_1_1 * tx_1_1) * 0.8660254037844386);
    aten_add[512 * blockIdx.x + threadIdx.x] = tx_1_1 * tx_1_1 + tz_1_1 * tz_1_1;
    aten_sub[512 * blockIdx.x + threadIdx.x] = ty_1_1 * ty_1_1 - (float)((double)(tx_1_1 * tx_1_1 + tz_1_1 * tz_1_1) * 0.5);
    aten_mul_2[512 * blockIdx.x + threadIdx.x] = (float)((double)(ty_1_1) * 1.732050807568877) * tz_1_1;
    aten_mul_1[512 * blockIdx.x + threadIdx.x] = (float)((double)(tx_1_1) * 1.732050807568877) * ty_1_1;
    aten_mul[512 * blockIdx.x + threadIdx.x] = (float)((double)(tx_1_1) * 1.732050807568877) * tz_1_1;
  }
}
}


</stderr_txt>
]]>




©2024 University of Washington
http://www.bakerlab.org