{ "schema_version": "1.0", "title": "AI inference benchmark: PyTorch baseline vs Intel OpenVINO", "run_id": "20260916-023938", "generated_at": "2026-09-16T01:48:17+00:00", "duration_seconds": 518.9, "hardware": { "cpu": { "model": "AMD Ryzen 5 7600X 6-Core Processor", "vendor": "AuthenticAMD", "architecture": "X86_64", "physical_cores": 6, "logical_processors": 12, "advertised_frequency_ghz": 4.7, "l2_cache_bytes": 6291456, "l3_cache_bytes": 33554432 }, "isa": { "sse4_2": true, "avx2": true, "fma": true, "avx512f": true, "avx512_vnni": true, "avx_vnni": false, "avx512_bf16": false, "amx_tile": false, "amx_int8": false, "amx_bf16": false }, "isa_summary": [ "AVX2", "AVX-512", "AVX-512 VNNI" ], "memory_total_gb": 31.2, "os": "Windows-11-10.0.26200-SP0", "is_intel": false, "openvino_cpu_device": { "full_device_name": "AMD Ryzen 5 7600X 6-Core Processor", "optimization_capabilities": [ "BF16", "WINOGRAD", "FP32", "INT8", "BIN", "EXPORT_IMPORT" ], "default_inference_precision": "", "available_devices": [ "CPU", "GPU" ] }, "memory": { "total_gb": 31.16, "available_gb": 12.04, "swap_total_gb": 10.5, "modules": [ { "slot": "DIMMA2", "capacity_gb": 16.0, "speed_mts": 4800, "configured_speed_mts": 5600, "type": "DDR5", "manufacturer": "Unknown", "part_number": "CMK32GX5M2B5600C36" }, { "slot": "DIMMB2", "capacity_gb": 16.0, "speed_mts": 4800, "configured_speed_mts": 5600, "type": "DDR5", "manufacturer": "Unknown", "part_number": "CMK32GX5M2B5600C36" } ], "configured_speed_mts": 5600, "type": "DDR5", "theoretical_bandwidth_gbs": 89.6, "channels_assumed": 2 }, "platform_notes": [ "Measured on a non-Intel x86-64 CPU. OpenVINO and oneDNN execute on any x86-64 CPU, but these are not Intel Xeon reference numbers. Re-run run_benchmarks.py on the target Xeon platform to regenerate them.", "No AMX (Advanced Matrix Extensions) detected. On 4th-gen Xeon Scalable or newer, OpenVINO INT8/BF16 kernels use AMX tiles and typically show larger gains than the VNNI path measured here." ] }, "software": { "python": "3.12.0", "numpy": "2.4.6", "torch": "2.14.0+cpu", "torchvision": "0.29.0+cpu", "transformers": "5.17.0", "openvino": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "nncf": "3.3.0", "psutil": "7.2.2", "openvino_build": "2026.3.1-22476-759c5a6ab8c-releases/2026/3" }, "methodology": { "isolation": "Each (model, configuration, batch size) runs in a fresh Python process that imports only its own framework; runs execute sequentially on an otherwise idle machine.", "timer": "time.perf_counter_ns around each synchronous forward call (pre-built inputs, no preprocessing inside the loop).", "warmup_iterations": 20, "timed_iterations_batch1": 200, "iteration_rule": "max(30, iterations // batch_size) calls and at least min_seconds, capped at max_seconds", "min_seconds": 3.0, "max_seconds": 60.0, "batch_sizes": [ 1, 8 ], "threads": 6, "thread_policy": "Both frameworks are given the same thread budget (physical cores): torch.set_num_threads for PyTorch, INFERENCE_NUM_THREADS with PERFORMANCE_HINT=LATENCY (1 stream) for OpenVINO.", "latency": "Per-call wall time in ms; p50 / p95 / p99 over the timed window.", "throughput": "batch_size x iterations / wall time of the timed window (synchronous, single stream).", "memory": "Peak resident set size of the isolated worker process: framework runtime + model + activations.", "cpu_utilization": "Process CPU time / wall time during the timed window, as % of all logical processors.", "accuracy": "Evaluated once per configuration at the smallest batch size on a fixed, seeded sample (vision: Imagenette validation scored with the full 1000-class argmax; NLP: SST-2 dev).", "eval_samples": 3925, "calibration_samples": 300, "openvino_precision_note": "INFERENCE_PRECISION_HINT is pinned to f32 for the FP32 and INT8 configurations so BF16-capable CPUs do not silently change the FP32 baseline comparison." }, "configs": [ { "id": "pytorch_fp32", "label": "PyTorch FP32", "short_label": "Baseline", "backend": "pytorch", "precision": "fp32", "role": "baseline", "description": "Stock PyTorch eager execution in FP32 using the published torchvision / Hugging Face weights. No graph compilation." }, { "id": "openvino_fp32", "label": "OpenVINO FP32", "short_label": "OpenVINO FP32", "backend": "openvino", "precision": "fp32", "role": "optimized", "description": "Same weights converted to OpenVINO IR and compiled for CPU: graph-level fusions, static shapes and oneDNN primitives. Inference precision pinned to f32." }, { "id": "openvino_int8", "label": "OpenVINO INT8", "short_label": "OpenVINO INT8", "backend": "openvino", "precision": "int8", "role": "optimized", "description": "OpenVINO IR quantized post-training with NNCF (8-bit weights and activations, calibrated on real samples). Executes on VNNI / AMX INT8 kernels where the CPU provides them." } ], "optimizations": [ { "id": "graph_compilation", "name": "OpenVINO graph compilation on oneDNN", "applies_to": [ "openvino_fp32", "openvino_int8", "openvino_bf16" ], "summary": "The PyTorch graph is captured once with ov.convert_model, optimised (Conv+Bias+ReLU fusion, constant folding, blocked memory layouts) and compiled for the CPU with static shapes; oneDNN JIT kernels execute it." }, { "id": "int8_quantization", "name": "Post-training INT8 quantization with NNCF", "applies_to": [ "openvino_int8" ], "summary": "nncf.quantize inserts 8-bit FakeQuantize ranges for weights and activations, calibrated on real samples. The CPU plugin lowers them to VNNI / AMX-INT8 dot products, cutting weight bytes ~4x and arithmetic cost per MAC." }, { "id": "latency_hint_threads", "name": "Latency performance hint and explicit thread budget", "applies_to": [ "openvino_fp32", "openvino_int8", "openvino_bf16" ], "summary": "PERFORMANCE_HINT=LATENCY runs a single inference stream pinned to the physical cores, the same thread budget given to the PyTorch baseline." } ], "models": [ { "id": "resnet50", "name": "ResNet-50", "family": "Convolutional network (50 layers)", "domain": "vision", "task": "Image classification / ImageNet-1k", "source": "torchvision resnet50 / ResNet50_Weights.IMAGENET1K_V2", "params_millions": 25.6, "input_names": [ "pixel_values" ], "input_shape": [ 3, 224, 224 ], "input_description": "3x224x224 RGB, ImageNet mean/std normalisation", "eval_dataset": "Imagenette v2 (160px) validation split", "accuracy_metric": "top-1 accuracy", "runs": [ { "model": "resnet50", "config": "pytorch_fp32", "batch_size": 1, "timestamp": "2026-09-16T01:39:55+00:00", "latency_ms": { "p50": 33.641, "p90": 38.326, "p95": 40.746, "p99": 65.629, "mean": 35.031, "std": 6.083, "min": 31.048, "max": 87.078 }, "throughput_ips": 28.54, "iterations": 200, "warmup": 20, "wall_seconds": 7.01, "threads": 6, "load_seconds": 2.72, "compile_seconds": 0.0, "memory_mb": { "process_start": 35.7, "after_model_load": 478.9, "peak": 497.6, "model_load_delta": 443.2 }, "cpu_utilization": { "process_pct_of_machine": 46.7, "process_cores_busy": 5.61, "process_pct_sampled": 46.9, "system_pct": 76.7, "logical_processors": 12, "peak_rss_sampled_bytes": 521789440, "samples": 63 }, "runtime": { "framework": "PyTorch", "version": "2.14.0+cpu", "execution": "eager, torch.inference_mode, FP32", "num_threads": 6, "onednn_available": true, "parallel_info": "ATen/Parallel:\n\tat::get_num_threads() : 6\n\tat::get_num_interop_threads() : 6\nOpenMP 2019\n\tomp_get_max_threads() : 6\nIntel(R) oneAPI Math Kernel Library Version 2026.1-Product Build 20260612 for Intel(R) 64 architecture applications\n\tmkl_get_max_threads() : 6\nIntel(R) MKL-DNN v3.12.0 (Git Hash 80afa71049cd69a3df32adcccb623b12cd7baa22)\nstd::thread::hardware_concurrency() : 12\nEnvironment variables:\n\tOMP_NUM_THREADS : [not set]\n\tMKL_NUM_THREADS : [not set]\nATen parallel backend: OpenMP\n" }, "accuracy": { "metric": "top-1 accuracy", "dataset": "Imagenette v2 (160px) validation split", "samples": 3925, "correct": 3151, "value": 0.8028, "eval_batch_size": 16, "seconds": 126.1, "value_10class": 0.9959, "note": "value = strict 1000-class ImageNet top-1; value_10class = argmax restricted to the 10 Imagenette classes" } }, { "model": "resnet50", "config": "pytorch_fp32", "batch_size": 8, "timestamp": "2026-09-16T01:42:16+00:00", "latency_ms": { "p50": 231.429, "p90": 242.354, "p95": 264.118, "p99": 279.768, "mean": 233.4, "std": 13.683, "min": 219.466, "max": 281.622 }, "throughput_ips": 34.28, "iterations": 30, "warmup": 20, "wall_seconds": 7.0, "threads": 6, "load_seconds": 2.65, "compile_seconds": 0.0, "memory_mb": { "process_start": 35.4, "after_model_load": 478.8, "peak": 520.6, "model_load_delta": 443.4 }, "cpu_utilization": { "process_pct_of_machine": 47.2, "process_cores_busy": 5.67, "process_pct_sampled": 47.4, "system_pct": 78.2, "logical_processors": 12, "peak_rss_sampled_bytes": 543141888, "samples": 64 }, "runtime": { "framework": "PyTorch", "version": "2.14.0+cpu", "execution": "eager, torch.inference_mode, FP32", "num_threads": 6, "onednn_available": true, "parallel_info": "ATen/Parallel:\n\tat::get_num_threads() : 6\n\tat::get_num_interop_threads() : 6\nOpenMP 2019\n\tomp_get_max_threads() : 6\nIntel(R) oneAPI Math Kernel Library Version 2026.1-Product Build 20260612 for Intel(R) 64 architecture applications\n\tmkl_get_max_threads() : 6\nIntel(R) MKL-DNN v3.12.0 (Git Hash 80afa71049cd69a3df32adcccb623b12cd7baa22)\nstd::thread::hardware_concurrency() : 12\nEnvironment variables:\n\tOMP_NUM_THREADS : [not set]\n\tMKL_NUM_THREADS : [not set]\nATen parallel backend: OpenMP\n" } }, { "model": "resnet50", "config": "openvino_fp32", "batch_size": 1, "timestamp": "2026-09-16T01:42:20+00:00", "latency_ms": { "p50": 15.364, "p90": 19.308, "p95": 21.495, "p99": 29.229, "mean": 16.378, "std": 5.306, "min": 12.185, "max": 64.176 }, "throughput_ips": 61.04, "iterations": 200, "warmup": 20, "wall_seconds": 3.28, "threads": 6, "load_seconds": 0.18, "compile_seconds": 0.16, "memory_mb": { "process_start": 35.4, "after_model_load": 59.6, "peak": 294.0, "model_load_delta": 24.2 }, "cpu_utilization": { "process_pct_of_machine": 43.5, "process_cores_busy": 5.22, "process_pct_sampled": 43.8, "system_pct": 73.1, "logical_processors": 12, "peak_rss_sampled_bytes": 308322304, "samples": 30 }, "runtime": { "framework": "OpenVINO", "version": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "device": "CPU", "ir": "optimized/models/resnet50/fp32/model.xml", "performance_hint": "LATENCY", "inference_precision": "", "num_streams": "1", "inference_num_threads": "6", "execution_devices": "['CPU']" }, "accuracy": { "metric": "top-1 accuracy", "dataset": "Imagenette v2 (160px) validation split", "samples": 3925, "correct": 3151, "value": 0.8028, "eval_batch_size": 16, "seconds": 54.6, "value_10class": 0.9959, "note": "value = strict 1000-class ImageNet top-1; value_10class = argmax restricted to the 10 Imagenette classes" } }, { "model": "resnet50", "config": "openvino_fp32", "batch_size": 8, "timestamp": "2026-09-16T01:43:21+00:00", "latency_ms": { "p50": 100.635, "p90": 116.404, "p95": 125.136, "p99": 128.519, "mean": 104.312, "std": 9.595, "min": 93.321, "max": 129.472 }, "throughput_ips": 76.69, "iterations": 30, "warmup": 20, "wall_seconds": 3.13, "threads": 6, "load_seconds": 0.16, "compile_seconds": 0.12, "memory_mb": { "process_start": 35.4, "after_model_load": 60.1, "peak": 329.2, "model_load_delta": 24.7 }, "cpu_utilization": { "process_pct_of_machine": 45.0, "process_cores_busy": 5.4, "process_pct_sampled": 44.7, "system_pct": 66.5, "logical_processors": 12, "peak_rss_sampled_bytes": 345145344, "samples": 28 }, "runtime": { "framework": "OpenVINO", "version": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "device": "CPU", "ir": "optimized/models/resnet50/fp32/model.xml", "performance_hint": "LATENCY", "inference_precision": "", "num_streams": "1", "inference_num_threads": "6", "execution_devices": "['CPU']" } }, { "model": "resnet50", "config": "openvino_int8", "batch_size": 1, "timestamp": "2026-09-16T01:43:25+00:00", "latency_ms": { "p50": 4.107, "p90": 5.234, "p95": 5.925, "p99": 7.925, "mean": 4.206, "std": 1.285, "min": 2.901, "max": 26.232 }, "throughput_ips": 237.56, "iterations": 713, "warmup": 20, "wall_seconds": 3.0, "threads": 6, "load_seconds": 0.15, "compile_seconds": 0.26, "memory_mb": { "process_start": 35.6, "after_model_load": 60.7, "peak": 153.5, "model_load_delta": 25.0 }, "cpu_utilization": { "process_pct_of_machine": 46.7, "process_cores_busy": 5.6, "process_pct_sampled": 46.5, "system_pct": 63.0, "logical_processors": 12, "peak_rss_sampled_bytes": 160968704, "samples": 27 }, "runtime": { "framework": "OpenVINO", "version": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "device": "CPU", "ir": "optimized/models/resnet50/int8/model.xml", "performance_hint": "LATENCY", "inference_precision": "", "num_streams": "1", "inference_num_threads": "6", "execution_devices": "['CPU']" }, "accuracy": { "metric": "top-1 accuracy", "dataset": "Imagenette v2 (160px) validation split", "samples": 3925, "correct": 3141, "value": 0.8003, "eval_batch_size": 16, "seconds": 19.6, "value_10class": 0.9954, "note": "value = strict 1000-class ImageNet top-1; value_10class = argmax restricted to the 10 Imagenette classes" } }, { "model": "resnet50", "config": "openvino_int8", "batch_size": 8, "timestamp": "2026-09-16T01:43:49+00:00", "latency_ms": { "p50": 26.517, "p90": 33.483, "p95": 34.983, "p99": 43.914, "mean": 27.901, "std": 3.963, "min": 23.394, "max": 48.189 }, "throughput_ips": 286.67, "iterations": 108, "warmup": 20, "wall_seconds": 3.01, "threads": 6, "load_seconds": 0.15, "compile_seconds": 0.25, "memory_mb": { "process_start": 35.5, "after_model_load": 60.7, "peak": 166.8, "model_load_delta": 25.2 }, "cpu_utilization": { "process_pct_of_machine": 46.1, "process_cores_busy": 5.54, "process_pct_sampled": 46.2, "system_pct": 68.4, "logical_processors": 12, "peak_rss_sampled_bytes": 174927872, "samples": 27 }, "runtime": { "framework": "OpenVINO", "version": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "device": "CPU", "ir": "optimized/models/resnet50/int8/model.xml", "performance_hint": "LATENCY", "inference_precision": "", "num_streams": "1", "inference_num_threads": "6", "execution_devices": "['CPU']" } } ], "conversion": { "converted_at": "2026-09-16T01:15:50+00:00", "convert_seconds": 1.4, "openvino_version": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "torch_version": "2.14.0+cpu", "quantization": { "quantize_seconds": 15.4, "calibration_samples": 300, "calibration_dataset": "Imagenette train split", "preset": "performance", "model_type": "cnn", "target_device": "CPU", "fake_quantize_nodes": 71, "nncf_version": "3.3.0" }, "fp32_weights_mb": 97.4, "int8_weights_mb": 24.5 }, "comparisons": { "1": { "openvino_fp32": { "latency_speedup_p50": 2.19, "latency_speedup_p95": 1.896, "latency_speedup_mean": 2.139, "throughput_gain": 2.139, "peak_memory_ratio": 0.591, "cpu_utilization_delta_pct": -3.2, "accuracy_delta_pp": 0.0, "prediction_agreement": 1.0 }, "openvino_int8": { "latency_speedup_p50": 8.191, "latency_speedup_p95": 6.877, "latency_speedup_mean": 8.329, "throughput_gain": 8.324, "peak_memory_ratio": 0.308, "cpu_utilization_delta_pct": 0.0, "accuracy_delta_pp": -0.25, "prediction_agreement": 0.9582 } }, "8": { "openvino_fp32": { "latency_speedup_p50": 2.3, "latency_speedup_p95": 2.111, "latency_speedup_mean": 2.238, "throughput_gain": 2.237, "peak_memory_ratio": 0.632, "cpu_utilization_delta_pct": -2.2 }, "openvino_int8": { "latency_speedup_p50": 8.728, "latency_speedup_p95": 7.55, "latency_speedup_mean": 8.365, "throughput_gain": 8.363, "peak_memory_ratio": 0.32, "cpu_utilization_delta_pct": -1.1 } } } }, { "id": "bert_base_sst2", "name": "BERT-base", "family": "Transformer encoder (12 layers, 768 hidden)", "domain": "nlp", "task": "Sentiment classification / SST-2", "source": "textattack/bert-base-uncased-SST-2 (Hugging Face)", "params_millions": 109.5, "input_names": [ "input_ids", "attention_mask", "token_type_ids" ], "seq_len": 128, "input_description": "sequence length 128 (padded), batch of token ids", "eval_dataset": "GLUE SST-2 dev split (872 sentences)", "accuracy_metric": "accuracy", "runs": [ { "model": "bert_base_sst2", "config": "pytorch_fp32", "batch_size": 1, "timestamp": "2026-09-16T01:44:12+00:00", "latency_ms": { "p50": 75.689, "p90": 80.437, "p95": 85.166, "p99": 106.957, "mean": 77.351, "std": 6.498, "min": 72.282, "max": 120.357 }, "throughput_ips": 12.93, "iterations": 200, "warmup": 20, "wall_seconds": 15.47, "threads": 6, "load_seconds": 6.28, "compile_seconds": 0.0, "memory_mb": { "process_start": 35.5, "after_model_load": 807.1, "peak": 828.8, "model_load_delta": 771.7 }, "cpu_utilization": { "process_pct_of_machine": 48.4, "process_cores_busy": 5.8, "process_pct_sampled": 48.5, "system_pct": 73.2, "logical_processors": 12, "peak_rss_sampled_bytes": 869007360, "samples": 141 }, "runtime": { "framework": "PyTorch", "version": "2.14.0+cpu", "execution": "eager, torch.inference_mode, FP32", "num_threads": 6, "onednn_available": true, "parallel_info": "ATen/Parallel:\n\tat::get_num_threads() : 6\n\tat::get_num_interop_threads() : 6\nOpenMP 2019\n\tomp_get_max_threads() : 6\nIntel(R) oneAPI Math Kernel Library Version 2026.1-Product Build 20260612 for Intel(R) 64 architecture applications\n\tmkl_get_max_threads() : 6\nIntel(R) MKL-DNN v3.12.0 (Git Hash 80afa71049cd69a3df32adcccb623b12cd7baa22)\nstd::thread::hardware_concurrency() : 12\nEnvironment variables:\n\tOMP_NUM_THREADS : [not set]\n\tMKL_NUM_THREADS : [not set]\nATen parallel backend: OpenMP\n" }, "accuracy": { "metric": "accuracy", "dataset": "GLUE SST-2 dev split (872 sentences)", "samples": 872, "correct": 806, "value": 0.9243, "eval_batch_size": 16, "seconds": 50.7 } }, { "model": "bert_base_sst2", "config": "pytorch_fp32", "batch_size": 8, "timestamp": "2026-09-16T01:45:34+00:00", "latency_ms": { "p50": 467.83, "p90": 493.26, "p95": 503.445, "p99": 513.18, "mean": 473.683, "std": 14.322, "min": 457.226, "max": 515.989 }, "throughput_ips": 16.89, "iterations": 30, "warmup": 20, "wall_seconds": 14.21, "threads": 6, "load_seconds": 6.21, "compile_seconds": 0.0, "memory_mb": { "process_start": 35.6, "after_model_load": 806.4, "peak": 873.7, "model_load_delta": 770.9 }, "cpu_utilization": { "process_pct_of_machine": 48.1, "process_cores_busy": 5.77, "process_pct_sampled": 48.2, "system_pct": 75.1, "logical_processors": 12, "peak_rss_sampled_bytes": 916144128, "samples": 130 }, "runtime": { "framework": "PyTorch", "version": "2.14.0+cpu", "execution": "eager, torch.inference_mode, FP32", "num_threads": 6, "onednn_available": true, "parallel_info": "ATen/Parallel:\n\tat::get_num_threads() : 6\n\tat::get_num_interop_threads() : 6\nOpenMP 2019\n\tomp_get_max_threads() : 6\nIntel(R) oneAPI Math Kernel Library Version 2026.1-Product Build 20260612 for Intel(R) 64 architecture applications\n\tmkl_get_max_threads() : 6\nIntel(R) MKL-DNN v3.12.0 (Git Hash 80afa71049cd69a3df32adcccb623b12cd7baa22)\nstd::thread::hardware_concurrency() : 12\nEnvironment variables:\n\tOMP_NUM_THREADS : [not set]\n\tMKL_NUM_THREADS : [not set]\nATen parallel backend: OpenMP\n" } }, { "model": "bert_base_sst2", "config": "openvino_fp32", "batch_size": 1, "timestamp": "2026-09-16T01:45:46+00:00", "latency_ms": { "p50": 43.232, "p90": 54.996, "p95": 57.29, "p99": 67.661, "mean": 45.368, "std": 7.333, "min": 36.489, "max": 84.795 }, "throughput_ips": 22.04, "iterations": 200, "warmup": 20, "wall_seconds": 9.07, "threads": 6, "load_seconds": 0.18, "compile_seconds": 0.38, "memory_mb": { "process_start": 35.4, "after_model_load": 61.4, "peak": 855.6, "model_load_delta": 26.0 }, "cpu_utilization": { "process_pct_of_machine": 45.7, "process_cores_busy": 5.48, "process_pct_sampled": 45.8, "system_pct": 69.4, "logical_processors": 12, "peak_rss_sampled_bytes": 897150976, "samples": 83 }, "runtime": { "framework": "OpenVINO", "version": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "device": "CPU", "ir": "optimized/models/bert_base_sst2/fp32/model.xml", "performance_hint": "LATENCY", "inference_precision": "", "num_streams": "1", "inference_num_threads": "6", "execution_devices": "['CPU']" }, "accuracy": { "metric": "accuracy", "dataset": "GLUE SST-2 dev split (872 sentences)", "samples": 872, "correct": 806, "value": 0.9243, "eval_batch_size": 16, "seconds": 37.1 } }, { "model": "bert_base_sst2", "config": "openvino_fp32", "batch_size": 8, "timestamp": "2026-09-16T01:46:39+00:00", "latency_ms": { "p50": 298.377, "p90": 323.063, "p95": 331.833, "p99": 338.575, "mean": 301.477, "std": 16.479, "min": 277.957, "max": 340.113 }, "throughput_ips": 26.54, "iterations": 30, "warmup": 20, "wall_seconds": 9.04, "threads": 6, "load_seconds": 0.16, "compile_seconds": 0.39, "memory_mb": { "process_start": 35.5, "after_model_load": 61.7, "peak": 871.9, "model_load_delta": 26.2 }, "cpu_utilization": { "process_pct_of_machine": 43.9, "process_cores_busy": 5.27, "process_pct_sampled": 44.0, "system_pct": 68.2, "logical_processors": 12, "peak_rss_sampled_bytes": 914296832, "samples": 83 }, "runtime": { "framework": "OpenVINO", "version": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "device": "CPU", "ir": "optimized/models/bert_base_sst2/fp32/model.xml", "performance_hint": "LATENCY", "inference_precision": "", "num_streams": "1", "inference_num_threads": "6", "execution_devices": "['CPU']" } }, { "model": "bert_base_sst2", "config": "openvino_int8", "batch_size": 1, "timestamp": "2026-09-16T01:46:44+00:00", "latency_ms": { "p50": 12.926, "p90": 18.047, "p95": 20.774, "p99": 23.334, "mean": 13.83, "std": 3.107, "min": 10.53, "max": 32.443 }, "throughput_ips": 72.28, "iterations": 217, "warmup": 20, "wall_seconds": 3.0, "threads": 6, "load_seconds": 0.17, "compile_seconds": 1.01, "memory_mb": { "process_start": 35.7, "after_model_load": 63.9, "peak": 421.7, "model_load_delta": 28.2 }, "cpu_utilization": { "process_pct_of_machine": 44.9, "process_cores_busy": 5.39, "process_pct_sampled": 45.2, "system_pct": 73.2, "logical_processors": 12, "peak_rss_sampled_bytes": 442171392, "samples": 27 }, "runtime": { "framework": "OpenVINO", "version": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "device": "CPU", "ir": "optimized/models/bert_base_sst2/int8/model.xml", "performance_hint": "LATENCY", "inference_precision": "", "num_streams": "1", "inference_num_threads": "6", "execution_devices": "['CPU']" }, "accuracy": { "metric": "accuracy", "dataset": "GLUE SST-2 dev split (872 sentences)", "samples": 872, "correct": 802, "value": 0.9197, "eval_batch_size": 16, "seconds": 10.0 } }, { "model": "bert_base_sst2", "config": "openvino_int8", "batch_size": 8, "timestamp": "2026-09-16T01:47:01+00:00", "latency_ms": { "p50": 86.248, "p90": 110.911, "p95": 118.583, "p99": 119.718, "mean": 90.102, "std": 12.06, "min": 76.322, "max": 119.899 }, "throughput_ips": 88.78, "iterations": 34, "warmup": 20, "wall_seconds": 3.06, "threads": 6, "load_seconds": 0.18, "compile_seconds": 0.97, "memory_mb": { "process_start": 35.5, "after_model_load": 65.1, "peak": 436.3, "model_load_delta": 29.6 }, "cpu_utilization": { "process_pct_of_machine": 44.2, "process_cores_busy": 5.3, "process_pct_sampled": 44.2, "system_pct": 69.2, "logical_processors": 12, "peak_rss_sampled_bytes": 457469952, "samples": 28 }, "runtime": { "framework": "OpenVINO", "version": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "device": "CPU", "ir": "optimized/models/bert_base_sst2/int8/model.xml", "performance_hint": "LATENCY", "inference_precision": "", "num_streams": "1", "inference_num_threads": "6", "execution_devices": "['CPU']" } } ], "conversion": { "converted_at": "2026-09-16T01:14:42+00:00", "convert_seconds": 1.8, "openvino_version": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "torch_version": "2.14.0+cpu", "quantization": { "quantize_seconds": 48.0, "calibration_samples": 300, "calibration_dataset": "GLUE SST-2 train split", "preset": "mixed (NNCF default for transformers)", "model_type": "transformer", "target_device": "CPU", "fake_quantize_nodes": 74, "nncf_version": "3.3.0" }, "fp32_weights_mb": 417.7, "int8_weights_mb": 105.5 }, "comparisons": { "1": { "openvino_fp32": { "latency_speedup_p50": 1.751, "latency_speedup_p95": 1.487, "latency_speedup_mean": 1.705, "throughput_gain": 1.705, "peak_memory_ratio": 1.032, "cpu_utilization_delta_pct": -2.7, "accuracy_delta_pp": 0.0, "prediction_agreement": 1.0 }, "openvino_int8": { "latency_speedup_p50": 5.856, "latency_speedup_p95": 4.1, "latency_speedup_mean": 5.593, "throughput_gain": 5.59, "peak_memory_ratio": 0.509, "cpu_utilization_delta_pct": -3.5, "accuracy_delta_pp": -0.46, "prediction_agreement": 0.9954 } }, "8": { "openvino_fp32": { "latency_speedup_p50": 1.568, "latency_speedup_p95": 1.517, "latency_speedup_mean": 1.571, "throughput_gain": 1.571, "peak_memory_ratio": 0.998, "cpu_utilization_delta_pct": -4.2 }, "openvino_int8": { "latency_speedup_p50": 5.424, "latency_speedup_p95": 4.246, "latency_speedup_mean": 5.257, "throughput_gain": 5.256, "peak_memory_ratio": 0.499, "cpu_utilization_delta_pct": -3.9 } } } }, { "id": "mobilenet_v3_large", "name": "MobileNetV3-Large", "family": "Depthwise-separable CNN (edge)", "domain": "vision", "task": "Image classification / ImageNet-1k", "source": "torchvision mobilenet_v3_large / MobileNet_V3_Large_Weights.IMAGENET1K_V2", "params_millions": 5.5, "input_names": [ "pixel_values" ], "input_shape": [ 3, 224, 224 ], "input_description": "3x224x224 RGB, ImageNet mean/std normalisation", "eval_dataset": "Imagenette v2 (160px) validation split", "accuracy_metric": "top-1 accuracy", "quantization_preset": "mixed", "runs": [ { "model": "mobilenet_v3_large", "config": "pytorch_fp32", "batch_size": 1, "timestamp": "2026-09-16T01:47:07+00:00", "latency_ms": { "p50": 10.609, "p90": 12.153, "p95": 13.241, "p99": 21.667, "mean": 11.162, "std": 3.149, "min": 9.21, "max": 53.098 }, "throughput_ips": 89.56, "iterations": 269, "warmup": 20, "wall_seconds": 3.0, "threads": 6, "load_seconds": 2.55, "compile_seconds": 0.0, "memory_mb": { "process_start": 35.5, "after_model_load": 327.3, "peak": 346.0, "model_load_delta": 291.8 }, "cpu_utilization": { "process_pct_of_machine": 47.1, "process_cores_busy": 5.65, "process_pct_sampled": 47.5, "system_pct": 77.5, "logical_processors": 12, "peak_rss_sampled_bytes": 352575488, "samples": 27 }, "runtime": { "framework": "PyTorch", "version": "2.14.0+cpu", "execution": "eager, torch.inference_mode, FP32", "num_threads": 6, "onednn_available": true, "parallel_info": "ATen/Parallel:\n\tat::get_num_threads() : 6\n\tat::get_num_interop_threads() : 6\nOpenMP 2019\n\tomp_get_max_threads() : 6\nIntel(R) oneAPI Math Kernel Library Version 2026.1-Product Build 20260612 for Intel(R) 64 architecture applications\n\tmkl_get_max_threads() : 6\nIntel(R) MKL-DNN v3.12.0 (Git Hash 80afa71049cd69a3df32adcccb623b12cd7baa22)\nstd::thread::hardware_concurrency() : 12\nEnvironment variables:\n\tOMP_NUM_THREADS : [not set]\n\tMKL_NUM_THREADS : [not set]\nATen parallel backend: OpenMP\n" }, "accuracy": { "metric": "top-1 accuracy", "dataset": "Imagenette v2 (160px) validation split", "samples": 3925, "correct": 2802, "value": 0.7139, "eval_batch_size": 16, "seconds": 26.4, "value_10class": 0.9911, "note": "value = strict 1000-class ImageNet top-1; value_10class = argmax restricted to the 10 Imagenette classes" } }, { "model": "mobilenet_v3_large", "config": "pytorch_fp32", "batch_size": 8, "timestamp": "2026-09-16T01:47:40+00:00", "latency_ms": { "p50": 36.324, "p90": 41.093, "p95": 41.931, "p99": 47.948, "mean": 37.237, "std": 3.123, "min": 33.758, "max": 53.998 }, "throughput_ips": 214.81, "iterations": 81, "warmup": 20, "wall_seconds": 3.02, "threads": 6, "load_seconds": 2.46, "compile_seconds": 0.0, "memory_mb": { "process_start": 35.6, "after_model_load": 326.9, "peak": 415.6, "model_load_delta": 291.3 }, "cpu_utilization": { "process_pct_of_machine": 47.9, "process_cores_busy": 5.75, "process_pct_sampled": 47.7, "system_pct": 77.2, "logical_processors": 12, "peak_rss_sampled_bytes": 435810304, "samples": 27 }, "runtime": { "framework": "PyTorch", "version": "2.14.0+cpu", "execution": "eager, torch.inference_mode, FP32", "num_threads": 6, "onednn_available": true, "parallel_info": "ATen/Parallel:\n\tat::get_num_threads() : 6\n\tat::get_num_interop_threads() : 6\nOpenMP 2019\n\tomp_get_max_threads() : 6\nIntel(R) oneAPI Math Kernel Library Version 2026.1-Product Build 20260612 for Intel(R) 64 architecture applications\n\tmkl_get_max_threads() : 6\nIntel(R) MKL-DNN v3.12.0 (Git Hash 80afa71049cd69a3df32adcccb623b12cd7baa22)\nstd::thread::hardware_concurrency() : 12\nEnvironment variables:\n\tOMP_NUM_THREADS : [not set]\n\tMKL_NUM_THREADS : [not set]\nATen parallel backend: OpenMP\n" } }, { "model": "mobilenet_v3_large", "config": "openvino_fp32", "batch_size": 1, "timestamp": "2026-09-16T01:47:45+00:00", "latency_ms": { "p50": 1.64, "p90": 2.669, "p95": 3.002, "p99": 4.264, "mean": 1.858, "std": 0.749, "min": 1.026, "max": 12.792 }, "throughput_ips": 537.38, "iterations": 1613, "warmup": 20, "wall_seconds": 3.0, "threads": 6, "load_seconds": 0.15, "compile_seconds": 0.15, "memory_mb": { "process_start": 35.4, "after_model_load": 60.5, "peak": 150.8, "model_load_delta": 25.1 }, "cpu_utilization": { "process_pct_of_machine": 44.6, "process_cores_busy": 5.35, "process_pct_sampled": 44.7, "system_pct": 73.0, "logical_processors": 12, "peak_rss_sampled_bytes": 158150656, "samples": 27 }, "runtime": { "framework": "OpenVINO", "version": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "device": "CPU", "ir": "optimized/models/mobilenet_v3_large/fp32/model.xml", "performance_hint": "LATENCY", "inference_precision": "", "num_streams": "1", "inference_num_threads": "6", "execution_devices": "['CPU']" }, "accuracy": { "metric": "top-1 accuracy", "dataset": "Imagenette v2 (160px) validation split", "samples": 3925, "correct": 2802, "value": 0.7139, "eval_batch_size": 16, "seconds": 11.4, "value_10class": 0.9911, "note": "value = strict 1000-class ImageNet top-1; value_10class = argmax restricted to the 10 Imagenette classes" } }, { "model": "mobilenet_v3_large", "config": "openvino_fp32", "batch_size": 8, "timestamp": "2026-09-16T01:48:00+00:00", "latency_ms": { "p50": 10.989, "p90": 14.329, "p95": 16.557, "p99": 24.779, "mean": 11.47, "std": 3.06, "min": 7.596, "max": 29.079 }, "throughput_ips": 697.09, "iterations": 262, "warmup": 20, "wall_seconds": 3.01, "threads": 6, "load_seconds": 0.16, "compile_seconds": 0.15, "memory_mb": { "process_start": 35.5, "after_model_load": 60.3, "peak": 181.3, "model_load_delta": 24.8 }, "cpu_utilization": { "process_pct_of_machine": 43.2, "process_cores_busy": 5.19, "process_pct_sampled": 42.8, "system_pct": 77.5, "logical_processors": 12, "peak_rss_sampled_bytes": 190124032, "samples": 27 }, "runtime": { "framework": "OpenVINO", "version": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "device": "CPU", "ir": "optimized/models/mobilenet_v3_large/fp32/model.xml", "performance_hint": "LATENCY", "inference_precision": "", "num_streams": "1", "inference_num_threads": "6", "execution_devices": "['CPU']" } }, { "model": "mobilenet_v3_large", "config": "openvino_int8", "batch_size": 1, "timestamp": "2026-09-16T01:48:04+00:00", "latency_ms": { "p50": 0.922, "p90": 1.674, "p95": 2.048, "p99": 2.718, "mean": 1.087, "std": 0.581, "min": 0.615, "max": 13.78 }, "throughput_ips": 918.21, "iterations": 2755, "warmup": 20, "wall_seconds": 3.0, "threads": 6, "load_seconds": 0.17, "compile_seconds": 0.34, "memory_mb": { "process_start": 35.5, "after_model_load": 62.1, "peak": 140.5, "model_load_delta": 26.6 }, "cpu_utilization": { "process_pct_of_machine": 44.8, "process_cores_busy": 5.38, "process_pct_sampled": 44.9, "system_pct": 71.9, "logical_processors": 12, "peak_rss_sampled_bytes": 147365888, "samples": 27 }, "runtime": { "framework": "OpenVINO", "version": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "device": "CPU", "ir": "optimized/models/mobilenet_v3_large/int8/model.xml", "performance_hint": "LATENCY", "inference_precision": "", "num_streams": "1", "inference_num_threads": "6", "execution_devices": "['CPU']" }, "accuracy": { "metric": "top-1 accuracy", "dataset": "Imagenette v2 (160px) validation split", "samples": 3925, "correct": 2744, "value": 0.6991, "eval_batch_size": 16, "seconds": 8.7, "value_10class": 0.988, "note": "value = strict 1000-class ImageNet top-1; value_10class = argmax restricted to the 10 Imagenette classes" } }, { "model": "mobilenet_v3_large", "config": "openvino_int8", "batch_size": 8, "timestamp": "2026-09-16T01:48:17+00:00", "latency_ms": { "p50": 5.169, "p90": 6.593, "p95": 7.29, "p99": 8.93, "mean": 5.432, "std": 1.216, "min": 3.895, "max": 22.28 }, "throughput_ips": 1471.48, "iterations": 552, "warmup": 20, "wall_seconds": 3.0, "threads": 6, "load_seconds": 0.16, "compile_seconds": 0.32, "memory_mb": { "process_start": 35.5, "after_model_load": 61.6, "peak": 152.2, "model_load_delta": 26.1 }, "cpu_utilization": { "process_pct_of_machine": 46.3, "process_cores_busy": 5.55, "process_pct_sampled": 46.1, "system_pct": 72.0, "logical_processors": 12, "peak_rss_sampled_bytes": 159629312, "samples": 27 }, "runtime": { "framework": "OpenVINO", "version": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "device": "CPU", "ir": "optimized/models/mobilenet_v3_large/int8/model.xml", "performance_hint": "LATENCY", "inference_precision": "", "num_streams": "1", "inference_num_threads": "6", "execution_devices": "['CPU']" } } ], "conversion": { "converted_at": "2026-09-16T01:39:29+00:00", "convert_seconds": 1.5, "openvino_version": "2026.3.1-22476-759c5a6ab8c-releases/2026/3", "torch_version": "2.14.0+cpu", "quantization": { "quantize_seconds": 7.5, "calibration_samples": 300, "calibration_dataset": "Imagenette train split", "preset": "mixed", "model_type": "cnn", "target_device": "CPU", "fake_quantize_nodes": 75, "fast_bias_correction": true, "nncf_version": "3.3.0" }, "fp32_weights_mb": 20.9, "int8_weights_mb": 9.7 }, "comparisons": { "1": { "openvino_fp32": { "latency_speedup_p50": 6.469, "latency_speedup_p95": 4.411, "latency_speedup_mean": 6.008, "throughput_gain": 6.0, "peak_memory_ratio": 0.436, "cpu_utilization_delta_pct": -2.5, "accuracy_delta_pp": 0.0, "prediction_agreement": 1.0 }, "openvino_int8": { "latency_speedup_p50": 11.507, "latency_speedup_p95": 6.465, "latency_speedup_mean": 10.269, "throughput_gain": 10.252, "peak_memory_ratio": 0.406, "cpu_utilization_delta_pct": -2.3, "accuracy_delta_pp": -1.48, "prediction_agreement": 0.8955 } }, "8": { "openvino_fp32": { "latency_speedup_p50": 3.305, "latency_speedup_p95": 2.533, "latency_speedup_mean": 3.246, "throughput_gain": 3.245, "peak_memory_ratio": 0.436, "cpu_utilization_delta_pct": -4.7 }, "openvino_int8": { "latency_speedup_p50": 7.027, "latency_speedup_p95": 5.752, "latency_speedup_mean": 6.855, "throughput_gain": 6.85, "peak_memory_ratio": 0.366, "cpu_utilization_delta_pct": -1.6 } } } } ], "headline": { "available": true, "config": "openvino_int8", "batch_size": 1, "model": "mobilenet_v3_large", "model_name": "MobileNetV3-Large", "latency_speedup_p50": 11.507, "latency_speedup_p95": 6.465, "throughput_gain": 10.252, "label": "11.5x lower latency with OpenVINO INT8", "geomean_int8_latency_speedup": 8.203, "geomean_int8_throughput_gain": 7.814, "geomean_int8_peak_memory_ratio": 0.399, "geomean_fp32_latency_speedup": 2.916, "worst_int8_accuracy_delta_pp": -1.48, "models_compared": 3 } }