| #include "ir3_nir.h" |
| |
| #include "nir.h" |
| #include "nir_builder.h" |
| #include "nir_search.h" |
| #include "nir_search_helpers.h" |
| |
| /* What follows is NIR algebraic transform code for the following 2 |
| * transforms: |
| * ('imul', 'a@32', 'b@32') => ('imadsh_mix16', 'b', 'a', ('imadsh_mix16', 'a', 'b', ('umul_low', 'a', 'b'))) |
| * ('iadd', ('imul24', 'a', 'b'), 'c') => ('imad24_ir3', 'a', 'b', 'c') |
| */ |
| |
| |
| static const nir_search_variable search0_0 = { |
| { nir_search_value_variable, 32 }, |
| 0, /* a */ |
| false, |
| nir_type_invalid, |
| NULL, |
| {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, |
| }; |
| |
| static const nir_search_variable search0_1 = { |
| { nir_search_value_variable, 32 }, |
| 1, /* b */ |
| false, |
| nir_type_invalid, |
| NULL, |
| {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, |
| }; |
| static const nir_search_expression search0 = { |
| { nir_search_value_expression, 32 }, |
| false, false, |
| 0, 1, |
| nir_op_imul, |
| { &search0_0.value, &search0_1.value }, |
| NULL, |
| }; |
| |
| /* replace0_0 -> search0_1 in the cache */ |
| |
| /* replace0_1 -> search0_0 in the cache */ |
| |
| /* replace0_2_0 -> search0_0 in the cache */ |
| |
| /* replace0_2_1 -> search0_1 in the cache */ |
| |
| /* replace0_2_2_0 -> search0_0 in the cache */ |
| |
| /* replace0_2_2_1 -> search0_1 in the cache */ |
| static const nir_search_expression replace0_2_2 = { |
| { nir_search_value_expression, 32 }, |
| false, false, |
| 0, 1, |
| nir_op_umul_low, |
| { &search0_0.value, &search0_1.value }, |
| NULL, |
| }; |
| static const nir_search_expression replace0_2 = { |
| { nir_search_value_expression, 32 }, |
| false, false, |
| -1, 1, |
| nir_op_imadsh_mix16, |
| { &search0_0.value, &search0_1.value, &replace0_2_2.value }, |
| NULL, |
| }; |
| static const nir_search_expression replace0 = { |
| { nir_search_value_expression, 32 }, |
| false, false, |
| -1, 1, |
| nir_op_imadsh_mix16, |
| { &search0_1.value, &search0_0.value, &replace0_2.value }, |
| NULL, |
| }; |
| |
| /* search1_0_0 -> search0_0 in the cache */ |
| |
| /* search1_0_1 -> search0_1 in the cache */ |
| static const nir_search_expression search1_0 = { |
| { nir_search_value_expression, 32 }, |
| false, false, |
| 1, 1, |
| nir_op_imul24, |
| { &search0_0.value, &search0_1.value }, |
| NULL, |
| }; |
| |
| static const nir_search_variable search1_1 = { |
| { nir_search_value_variable, 32 }, |
| 2, /* c */ |
| false, |
| nir_type_invalid, |
| NULL, |
| {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, |
| }; |
| static const nir_search_expression search1 = { |
| { nir_search_value_expression, 32 }, |
| false, false, |
| 0, 2, |
| nir_op_iadd, |
| { &search1_0.value, &search1_1.value }, |
| NULL, |
| }; |
| |
| /* replace1_0 -> search0_0 in the cache */ |
| |
| /* replace1_1 -> search0_1 in the cache */ |
| |
| /* replace1_2 -> search1_1 in the cache */ |
| static const nir_search_expression replace1 = { |
| { nir_search_value_expression, 32 }, |
| false, false, |
| 0, 1, |
| nir_op_imad24_ir3, |
| { &search0_0.value, &search0_1.value, &search1_1.value }, |
| NULL, |
| }; |
| |
| |
| static const struct transform ir3_nir_lower_imul_state2_xforms[] = { |
| { &search0, &replace0.value, 0 }, |
| }; |
| static const struct transform ir3_nir_lower_imul_state4_xforms[] = { |
| { &search1, &replace1.value, 0 }, |
| }; |
| |
| static const struct per_op_table ir3_nir_lower_imul_table[nir_num_search_ops] = { |
| [nir_op_imul] = { |
| .filter = (uint16_t []) { |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| }, |
| |
| .num_filtered_states = 1, |
| .table = (uint16_t []) { |
| |
| 2, |
| }, |
| }, |
| [nir_op_iadd] = { |
| .filter = (uint16_t []) { |
| 0, |
| 0, |
| 0, |
| 1, |
| 0, |
| }, |
| |
| .num_filtered_states = 2, |
| .table = (uint16_t []) { |
| |
| 0, |
| 4, |
| 4, |
| 4, |
| }, |
| }, |
| [nir_op_imul24] = { |
| .filter = (uint16_t []) { |
| 0, |
| 0, |
| 0, |
| 0, |
| 0, |
| }, |
| |
| .num_filtered_states = 1, |
| .table = (uint16_t []) { |
| |
| 3, |
| }, |
| }, |
| }; |
| |
| const struct transform *ir3_nir_lower_imul_transforms[] = { |
| NULL, |
| NULL, |
| ir3_nir_lower_imul_state2_xforms, |
| NULL, |
| ir3_nir_lower_imul_state4_xforms, |
| }; |
| |
| const uint16_t ir3_nir_lower_imul_transform_counts[] = { |
| 0, |
| 0, |
| (uint16_t)ARRAY_SIZE(ir3_nir_lower_imul_state2_xforms), |
| 0, |
| (uint16_t)ARRAY_SIZE(ir3_nir_lower_imul_state4_xforms), |
| }; |
| |
| bool |
| ir3_nir_lower_imul(nir_shader *shader) |
| { |
| bool progress = false; |
| bool condition_flags[1]; |
| const nir_shader_compiler_options *options = shader->options; |
| const shader_info *info = &shader->info; |
| (void) options; |
| (void) info; |
| |
| condition_flags[0] = true; |
| |
| nir_foreach_function(function, shader) { |
| if (function->impl) { |
| progress |= nir_algebraic_impl(function->impl, condition_flags, |
| ir3_nir_lower_imul_transforms, |
| ir3_nir_lower_imul_transform_counts, |
| ir3_nir_lower_imul_table); |
| } |
| } |
| |
| return progress; |
| } |
| |