diff --git a/.benchmarks/Darwin-CPython-3.11-64bit/0001_8a7432129b462fb2ee4297789fbd15a3b88b07d1_20241121_133526_uncommited-changes.json b/.benchmarks/Darwin-CPython-3.11-64bit/0001_8a7432129b462fb2ee4297789fbd15a3b88b07d1_20241121_133526_uncommited-changes.json deleted file mode 100644 index c47de40..0000000 --- a/.benchmarks/Darwin-CPython-3.11-64bit/0001_8a7432129b462fb2ee4297789fbd15a3b88b07d1_20241121_133526_uncommited-changes.json +++ /dev/null @@ -1,148 +0,0 @@ -{ - "machine_info": { - "node": "Woile-MacBook-Pro.local", - "processor": "arm", - "machine": "arm64", - "python_compiler": "Clang 16.0.6 ", - "python_implementation": "CPython", - "python_implementation_version": "3.11.10", - "python_version": "3.11.10", - "python_build": [ - "main", - "Sep 7 2024 01:03:31" - ], - "release": "24.1.0", - "system": "Darwin", - "cpu": { - "python_version": "3.11.10.final.0 (64 bit)", - "cpuinfo_version": [ - 9, - 0, - 0 - ], - "cpuinfo_version_string": "9.0.0", - "arch": "ARM_8", - "bits": 64, - "count": 12, - "arch_string_raw": "arm64", - "brand_raw": "Apple M3 Pro" - } - }, - "commit_info": { - "id": "8a7432129b462fb2ee4297789fbd15a3b88b07d1", - "time": "2024-11-20T17:03:56+01:00", - "author_time": "2024-11-19T16:40:26+01:00", - "dirty": true, - "project": "kstreams", - "branch": "(detached head)" - }, - "benchmarks": [ - { - "group": null, - "name": "test_processing_single_consumer_record", - "fullname": "tests/test_benchmarks.py::test_processing_single_consumer_record", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false - }, - "stats": { - "min": 4.229199839755893e-05, - "max": 0.0008245829994848464, - "mean": 7.612135620897753e-05, - "stddev": 2.585269375808259e-05, - "rounds": 2673, - "median": 7.541700324509293e-05, - "iqr": 3.001999812113354e-05, - "q1": 5.990650151943555e-05, - "q3": 8.992649964056909e-05, - "iqr_outliers": 17, - "stddev_outliers": 448, - "outliers": "448;17", - "ld15iqr": 4.229199839755893e-05, - "hd15iqr": 0.00013625000065076165, - "ops": 13136.917808645965, - "total": 0.20347238514659693, - "iterations": 1 - } - }, - { - "group": null, - "name": "test_inject_all", - "fullname": "tests/test_benchmarks.py::test_inject_all", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false - }, - "stats": { - "min": 4.337500286055729e-05, - "max": 0.021463125001901062, - "mean": 0.0002409296128066245, - "stddev": 0.00032767216641994395, - "rounds": 14954, - "median": 0.00022097950022725854, - "iqr": 0.000182791001861915, - "q1": 0.00013158399815438315, - "q3": 0.00031437500001629815, - "iqr_outliers": 161, - "stddev_outliers": 172, - "outliers": "172;161", - "ld15iqr": 4.337500286055729e-05, - "hd15iqr": 0.000590749998082174, - "ops": 4150.5898272564045, - "total": 3.602861429910263, - "iterations": 1 - } - }, - { - "group": null, - "name": "test_consume_many", - "fullname": "tests/test_benchmarks.py::test_consume_many", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false - }, - "stats": { - "min": 0.0007587500003864989, - "max": 0.0012585000004037283, - "mean": 0.0008135390617422691, - "stddev": 3.9826845749555304e-05, - "rounds": 955, - "median": 0.0008057500017457642, - "iqr": 4.940624876326183e-05, - "q1": 0.0007849169996916316, - "q3": 0.0008343232484548935, - "iqr_outliers": 15, - "stddev_outliers": 215, - "outliers": "215;15", - "ld15iqr": 0.0007587500003864989, - "hd15iqr": 0.00091229100144119, - "ops": 1229.1972777046594, - "total": 0.776929803963867, - "iterations": 1 - } - } - ], - "datetime": "2024-11-21T13:35:33.270192+00:00", - "version": "5.1.0" -} \ No newline at end of file diff --git a/.benchmarks/Darwin-CPython-3.11-64bit/0001_a21803ba4cdbcba27f6b9b49880cc8615996c778_20241126_143811_uncommited-changes.json b/.benchmarks/Darwin-CPython-3.11-64bit/0001_a21803ba4cdbcba27f6b9b49880cc8615996c778_20241126_143811_uncommited-changes.json new file mode 100644 index 0000000..1d8e58e --- /dev/null +++ b/.benchmarks/Darwin-CPython-3.11-64bit/0001_a21803ba4cdbcba27f6b9b49880cc8615996c778_20241126_143811_uncommited-changes.json @@ -0,0 +1,148 @@ +{ + "machine_info": { + "node": "mac.home", + "processor": "arm", + "machine": "arm64", + "python_compiler": "Clang 16.0.6 ", + "python_implementation": "CPython", + "python_implementation_version": "3.11.10", + "python_version": "3.11.10", + "python_build": [ + "main", + "Sep 7 2024 01:03:31" + ], + "release": "24.1.0", + "system": "Darwin", + "cpu": { + "python_version": "3.11.10.final.0 (64 bit)", + "cpuinfo_version": [ + 9, + 0, + 0 + ], + "cpuinfo_version_string": "9.0.0", + "arch": "ARM_8", + "bits": 64, + "count": 12, + "arch_string_raw": "arm64", + "brand_raw": "Apple M3 Pro" + } + }, + "commit_info": { + "id": "a21803ba4cdbcba27f6b9b49880cc8615996c778", + "time": "2024-11-22T12:07:10+00:00", + "author_time": "2024-11-22T12:07:10+00:00", + "dirty": true, + "project": "kstreams", + "branch": "fix/benches" + }, + "benchmarks": [ + { + "group": null, + "name": "test_startup_and_processing_single_consumer_record", + "fullname": "tests/test_benchmarks.py::test_startup_and_processing_single_consumer_record", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false + }, + "stats": { + "min": 4.300000728107989e-05, + "max": 0.013382542005274445, + "mean": 0.00012213260948737922, + "stddev": 0.00020234487853902205, + "rounds": 5313, + "median": 0.00011308302055113018, + "iqr": 7.325000478886068e-05, + "q1": 7.545799599029124e-05, + "q3": 0.00014870800077915192, + "iqr_outliers": 94, + "stddev_outliers": 75, + "outliers": "75;94", + "ld15iqr": 4.300000728107989e-05, + "hd15iqr": 0.00026529200840741396, + "ops": 8187.8214524134655, + "total": 0.6488905542064458, + "iterations": 1 + } + }, + { + "group": null, + "name": "test_startup_and_inject_all", + "fullname": "tests/test_benchmarks.py::test_startup_and_inject_all", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false + }, + "stats": { + "min": 4.454201553016901e-05, + "max": 0.021896749996813014, + "mean": 0.00022779209596639227, + "stddev": 0.00029202890387583687, + "rounds": 15114, + "median": 0.00021839600231032819, + "iqr": 0.00018137501319870353, + "q1": 0.00013420797768048942, + "q3": 0.00031558299087919295, + "iqr_outliers": 18, + "stddev_outliers": 29, + "outliers": "29;18", + "ld15iqr": 4.454201553016901e-05, + "hd15iqr": 0.0006014160171616822, + "ops": 4389.967947560116, + "total": 3.4428497384360526, + "iterations": 1 + } + }, + { + "group": null, + "name": "test_consume_many", + "fullname": "tests/test_benchmarks.py::test_consume_many", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": false + }, + "stats": { + "min": 0.0007284579914994538, + "max": 0.0027474589878693223, + "mean": 0.0007670833579878962, + "stddev": 7.537617304008719e-05, + "rounds": 1146, + "median": 0.0007534790056524798, + "iqr": 3.520897007547319e-05, + "q1": 0.0007426660158671439, + "q3": 0.0007778749859426171, + "iqr_outliers": 45, + "stddev_outliers": 36, + "outliers": "36;45", + "ld15iqr": 0.0007284579914994538, + "hd15iqr": 0.0008308329852297902, + "ops": 1303.6392845531907, + "total": 0.879077528254129, + "iterations": 1 + } + } + ], + "datetime": "2024-11-26T14:38:18.502271+00:00", + "version": "5.1.0" +} \ No newline at end of file diff --git a/.benchmarks/Linux-CPython-3.10-64bit/0001_0192fcc771e53d2e05903d956c45a99014aac635_20241122_120704.json b/.benchmarks/Linux-CPython-3.10-64bit/0001_0192fcc771e53d2e05903d956c45a99014aac635_20241122_120704.json deleted file mode 100644 index 1bd29da..0000000 --- a/.benchmarks/Linux-CPython-3.10-64bit/0001_0192fcc771e53d2e05903d956c45a99014aac635_20241122_120704.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "machine_info": { - "node": "fv-az1110-415", - "processor": "x86_64", - "machine": "x86_64", - "python_compiler": "GCC 11.4.0", - "python_implementation": "CPython", - "python_implementation_version": "3.10.15", - "python_version": "3.10.15", - "python_build": [ - "main", - "Sep 9 2024 03:02:45" - ], - "release": "6.5.0-1025-azure", - "system": "Linux", - "cpu": { - "python_version": "3.10.15.final.0 (64 bit)", - "cpuinfo_version": [ - 9, - 0, - 0 - ], - "cpuinfo_version_string": "9.0.0", - "arch": "X86_64", - "bits": 64, - "count": 4, - "arch_string_raw": "x86_64", - "vendor_id_raw": "AuthenticAMD", - "brand_raw": "AMD EPYC 7763 64-Core Processor", - "hz_advertised_friendly": "2.4454 GHz", - "hz_actual_friendly": "2.4454 GHz", - "hz_advertised": [ - 2445436000, - 0 - ], - "hz_actual": [ - 2445436000, - 0 - ], - "stepping": 1, - "model": 1, - "family": 25, - "flags": [ - "3dnowext", - "3dnowprefetch", - "abm", - "adx", - "aes", - "aperfmperf", - "apic", - "arat", - "avx", - "avx2", - "bmi1", - "bmi2", - "clflush", - "clflushopt", - "clwb", - "clzero", - "cmov", - "cmp_legacy", - "constant_tsc", - "cpuid", - "cr8_legacy", - "cx16", - "cx8", - "de", - "decodeassists", - "erms", - "extd_apicid", - "f16c", - "flushbyasid", - "fma", - "fpu", - "fsgsbase", - "fsrm", - "fxsr", - "fxsr_opt", - "ht", - "hypervisor", - "invpcid", - "invpcid_single", - "lahf_lm", - "lm", - "mca", - "mce", - "misalignsse", - "mmx", - "mmxext", - "movbe", - "msr", - "mtrr", - "nonstop_tsc", - "nopl", - "npt", - "nrip_save", - "nx", - "osvw", - "osxsave", - "pae", - "pat", - "pausefilter", - "pcid", - "pclmulqdq", - "pdpe1gb", - "pfthreshold", - "pge", - "pni", - "popcnt", - "pse", - "pse36", - "rdpid", - "rdpru", - "rdrand", - "rdrnd", - "rdseed", - "rdtscp", - "rep_good", - "sep", - "sha", - "sha_ni", - "smap", - "smep", - "sse", - "sse2", - "sse4_1", - "sse4_2", - "sse4a", - "ssse3", - "svm", - "syscall", - "topoext", - "tsc", - "tsc_reliable", - "tsc_scale", - "umip", - "v_vmsave_vmload", - "vaes", - "vmcb_clean", - "vme", - "vmmcall", - "vpclmulqdq", - "xgetbv1", - "xsave", - "xsavec", - "xsaveerptr", - "xsaveopt", - "xsaves" - ], - "l3_cache_size": 524288, - "l2_cache_size": 1048576, - "l1_data_cache_size": 65536, - "l1_instruction_cache_size": 65536, - "l2_cache_line_size": 512, - "l2_cache_associativity": 6 - } - }, - "commit_info": { - "id": "0192fcc771e53d2e05903d956c45a99014aac635", - "time": "2024-11-22T12:06:08Z", - "author_time": "2024-11-22T12:06:08Z", - "dirty": false, - "project": "kstreams", - "branch": "master" - }, - "benchmarks": [ - { - "group": null, - "name": "test_processing_single_consumer_record", - "fullname": "tests/test_benchmarks.py::test_processing_single_consumer_record", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false - }, - "stats": { - "min": 8.448899998825254e-05, - "max": 0.021019971000015403, - "mean": 0.00015135989433467492, - "stddev": 0.0003911976066316952, - "rounds": 2877, - "median": 0.0001405250000061642, - "iqr": 4.718549999438437e-05, - "q1": 0.00012100300000383868, - "q3": 0.00016818849999822305, - "iqr_outliers": 15, - "stddev_outliers": 6, - "outliers": "6;15", - "ld15iqr": 8.448899998825254e-05, - "hd15iqr": 0.0002412850000155231, - "ops": 6606.769939921335, - "total": 0.4354624160008598, - "iterations": 1 - } - }, - { - "group": null, - "name": "test_inject_all", - "fullname": "tests/test_benchmarks.py::test_inject_all", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false - }, - "stats": { - "min": 8.377799997560942e-05, - "max": 0.03924704400000678, - "mean": 0.00023857912354777408, - "stddev": 0.0006463593905425304, - "rounds": 7058, - "median": 0.00022919649998698333, - "iqr": 0.00014046499998698891, - "q1": 0.0001549110000098608, - "q3": 0.0002953759999968497, - "iqr_outliers": 43, - "stddev_outliers": 3, - "outliers": "3;43", - "ld15iqr": 8.377799997560942e-05, - "hd15iqr": 0.0005261910000058379, - "ops": 4191.481572777912, - "total": 1.6838914540001895, - "iterations": 1 - } - }, - { - "group": null, - "name": "test_consume_many", - "fullname": "tests/test_benchmarks.py::test_consume_many", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": false - }, - "stats": { - "min": 0.00247113199998239, - "max": 0.003942762000008315, - "mean": 0.002562978832807956, - "stddev": 0.00014576839439977497, - "rounds": 317, - "median": 0.002541995000001407, - "iqr": 5.213450000951525e-05, - "q1": 0.0025168279999974175, - "q3": 0.0025689625000069327, - "iqr_outliers": 14, - "stddev_outliers": 6, - "outliers": "6;14", - "ld15iqr": 0.00247113199998239, - "hd15iqr": 0.0026487620000068546, - "ops": 390.170994469126, - "total": 0.812464290000122, - "iterations": 1 - } - } - ], - "datetime": "2024-11-22T12:07:10.470279+00:00", - "version": "5.1.0" -} \ No newline at end of file diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index 6721b02..592a99e 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -1,3 +1,25 @@ +""" +This file contians benchmarks for kstreams. + +Interpreting the data: + +- Name: Name of the test or function being benchmarked. +- Min: The shortest execution time recorded across all runs. +- Max: The longest execution time recorded. +- Mean: The average execution time across all runs. +- StdDev: The standard deviation of execution times, representing the variability of the + measurements. A low value indicates consistent performance. +- Median: The middle value when all execution times are sorted. +- IQR (Interquartile Range): The range between the 25th and 75th percentile of execution + times. It’s a robust measure of variability that’s less sensitive to outliers. +- Outliers: Measurements that significantly differ from others. + E.g., 5;43 means 5 mild and 43 extreme outliers. +- OPS (Operations Per Second): How many times the function could execute + in one second (calculated as 1 / Mean). +- Rounds: The number of times the test was run. +- Iterations: Number of iterations per round. +""" + from typing import Callable, List from kstreams.engine import StreamEngine @@ -5,7 +27,7 @@ from kstreams.types import ConsumerRecord, Send -async def bench_processing_single_consumer_record( +async def bench_startup_and_processing_single_consumer_record( stream_engine: StreamEngine, cr: ConsumerRecord, ): @@ -24,7 +46,7 @@ async def consume_bench(cr: ConsumerRecord): return r -async def bench_inject_all_deps( +async def bench_startup_and_inject_all_deps( stream_engine: StreamEngine, cr: ConsumerRecord, ): @@ -44,7 +66,7 @@ async def consume_bench(cr: ConsumerRecord, stream: Stream, send: Send): async def bench_consume_many( - stream_engine: StreamEngine, + consume_bench: Stream, crs: List[ConsumerRecord], ): """ @@ -53,16 +75,11 @@ async def bench_consume_many( This function benchmarks the processing of a single consumer record by using a stream engine to consume the record from a specified topic. """ - - @stream_engine.stream("local--kstreams") - async def consume_bench(cr: ConsumerRecord, stream: Stream, send: Send): - return cr.topic - for cr in crs: _ = await consume_bench.func(cr) -def test_processing_single_consumer_record( +def test_startup_and_processing_single_consumer_record( stream_engine: StreamEngine, consumer_record_factory: Callable[..., ConsumerRecord], aio_benchmark, @@ -70,13 +87,13 @@ def test_processing_single_consumer_record( cr = consumer_record_factory() aio_benchmark( - bench_processing_single_consumer_record, + bench_startup_and_processing_single_consumer_record, stream_engine, cr, ) -def test_inject_all( +def test_startup_and_inject_all( stream_engine: StreamEngine, consumer_record_factory: Callable[..., ConsumerRecord], aio_benchmark, @@ -84,7 +101,7 @@ def test_inject_all( cr = consumer_record_factory() aio_benchmark( - bench_inject_all_deps, + bench_startup_and_inject_all_deps, stream_engine, cr, ) @@ -95,10 +112,15 @@ def test_consume_many( consumer_record_factory: Callable[..., ConsumerRecord], aio_benchmark, ): + """In this case, the startup happens outside the benchmark.""" crs = [consumer_record_factory() for _ in range(1000)] + @stream_engine.stream("local--kstreams") + async def consume_bench(cr: ConsumerRecord, stream: Stream, send: Send): + return cr.topic + aio_benchmark( bench_consume_many, - stream_engine, + consume_bench, crs, )