fix CI, update problem set

Signed-off-by: Max Dawkins <[email protected]>

.github/workflows/run_bench.yml

@@ -56,7 +56,7 @@ jobs:
         run: |
           source bench_venv/bin/activate
           python convbench/conv_bench.py --roofline results/iree_conv.csv --plot results/iree_conv_i8.png --dtype i8
-          python convbench/conv_bench.py --roofline results/iree_conv.csv --plot results/iree_conv_f32.png --dtype f32
+          python convbench/conv_bench.py --roofline results/iree_conv.csv --plot results/iree_conv_f16.png --dtype f16
           python convbench/conv_bench.py --roofline results/iree_attention.csv --plot results/iree_attention_fp16.png --dtype f16
           python convbench/conv_bench.py --roofline results/iree_attention.csv --plot results/iree_attention_fp8.png --dtype f8E4M3FNUZ
           python convbench/conv_bench.py --roofline results/iree_gemm.csv --plot results/iree_gemm.png

common_tools/utils/bench_utils.py

@@ -144,7 +144,7 @@ def roofline(results=None, out=None, batch=None, dtype=None, model=None, **kwarg
         with open(result_file.strip(), mode='r') as csvfile:
             reader = csv.DictReader(csvfile)
             for row in reader:
-                row = {k: float(v) if k in ['index', 'mean_microseconds', 'arithmetic_intensity', 'tflops'] else v for k, v in row.items()}
+                row = {k: float(v) if k in ['index', 'mean_microseconds', 'arithmetic_intensity', 'tflops', 'roofline_tflops', 'roofline_percent'] else v for k, v in row.items()}
                 row['ok'] = True if 'ok' not in row else row['ok'] == 'True'
                 data.append(row)
         if batch:

convbench/conv_bench.py

@@ -50,8 +50,8 @@ def compile_conv(tag, config, kernel_dir, vmfb_dir, extra_compiler_args):
         roofline(args.roofline, args.plot, args.batch, args.dtype, args.model)
         sys.exit()
 
-    configs = get_conv_test_configs()
-    # configs = get_conv_configs()
+    # configs = get_conv_test_configs()
+    configs = get_conv_configs()
     print(f"Generated {len(configs)} conv configs.")
 
     num_cpus = max(1, cpu_count() - 20)

convbench/problems.py

@@ -3,7 +3,7 @@
 
 def unet_sweep(op: str, input_dtype: str, output_dtype: str) -> list[ConvConfig]:
     configs = []
-    for B in [1, 2, 4]:
+    for B in [1, 2, 4, 8]:
         configs.append(ConvConfig(B, 128, 128, 16, 3, 3, 320, 1, op, input_dtype, output_dtype))
         configs.append(ConvConfig(B, 128, 128, 320, 3, 3, 320, 1, op, input_dtype, output_dtype))
         configs.append(ConvConfig(B, 64, 64, 320, 3, 3, 320, 2, op, input_dtype, output_dtype))
@@ -35,7 +35,7 @@ def unet_sweep(op: str, input_dtype: str, output_dtype: str) -> list[ConvConfig]
 
 def resnet_sweep(op: str, input_dtype: str, output_dtype: str) -> list[ConvConfig]:
     configs = []
-    for B in [1, 2, 4, 8, 16, 32, 48]:
+    for B in [1, 2, 4, 8]:
         configs.append(ConvConfig(B, 112, 112, 64, 7, 7, 3, 2, op, input_dtype, output_dtype))
         configs.append(ConvConfig(B, 56, 56, 64, 3, 3, 64, 1, op, input_dtype, output_dtype))
         configs.append(ConvConfig(B, 28, 28, 128, 3, 3, 128, 2, op, input_dtype, output_dtype))
@@ -55,13 +55,17 @@ def get_conv_configs() -> list[tuple[str, ConvConfig]]:
     # Resnet
     resnet_configs = []
     resnet_configs += resnet_sweep("conv_2d_nhwc_hwcf", "f16", "f32")
-    resnet_configs += resnet_sweep("conv_2d_nhwc_hwcf_q", "i8", "i32")
+    resnet_configs += resnet_sweep("conv_2d_nhwc_hwcf", "i8", "i32")
+    resnet_configs += resnet_sweep("conv_2d_nchw_fchw", "f16", "f32")
+    resnet_configs += resnet_sweep("conv_2d_nchw_fchw", "i8", "i32")
     configs += [("resnet", x) for x in resnet_configs]
 
     # Unet
     unet_configs = []
     unet_configs += unet_sweep("conv_2d_nhwc_hwcf", "f16", "f32")
-    unet_configs += unet_sweep("conv_2d_nhwc_hwcf_q", "i8", "i32")
+    unet_configs += unet_sweep("conv_2d_nhwc_hwcf", "i8", "i32")
+    unet_configs += unet_sweep("conv_2d_nchw_fchw", "f16", "f32")
+    unet_configs += unet_sweep("conv_2d_nchw_fchw", "i8", "i32")
     configs += [("unet", x) for x in unet_configs]
 
     return configs
@@ -70,9 +74,16 @@ def get_conv_configs() -> list[tuple[str, ConvConfig]]:
 def get_conv_test_configs() -> list[tuple[str, ConvConfig]]:
     configs: list[tuple[str, ConvConfig]] = []
 
+    resnet_configs = []
+    # resnet_configs += resnet_sweep("conv_2d_nhwc_hwcf", "f16", "f32")
+    # resnet_configs += resnet_sweep("conv_2d_nhwc_hwcf", "i8", "i32")
+    # resnet_configs += resnet_sweep("conv_2d_nchw_fchw", "f16", "f32")
+    # resnet_configs += resnet_sweep("conv_2d_nchw_fchw", "i8", "i32")
+    configs += [("resnet", x) for x in resnet_configs]
+
     unet_configs = []
-    unet_configs.append(ConvConfig(1,128,128,16,3,3,320,1, "conv_2d_nhwc_hwcf_q", "i8", "i32"))
-    unet_configs.append(ConvConfig(1,32,32,640,1,1,1280,1, "conv_2d_nhwc_hwcf_q", "i8", "i32"))
+    # unet_configs.append(ConvConfig(1,128,128,16,3,3,320,1, "conv_2d_nhwc_hwcf_q", "i8", "i32"))
+    # unet_configs.append(ConvConfig(1,32,32,640,1,1,1280,1, "conv_2d_nhwc_hwcf_q", "i8", "i32"))
 
     configs += [("unet", x) for x in unet_configs]