perf stats; don't merge perf branch lol
- SHA
4ced185a37ab69a22bfadf2298b2a525c0a5ba0b- Parents
-
103865f - Tree
bc0861c
4ced185
4ced185a37ab69a22bfadf2298b2a525c0a5ba0b103865f
bc0861c| Status | File | + | - |
|---|---|---|---|
| A |
src/shtick/bench-intensive-opt-20250602-223501.out
|
437 | 0 |
| A |
src/shtick/bench-intensive-std-20250602-223632.out
|
310 | 0 |
| A |
src/shtick/bench.py
|
724 | 0 |
| A |
src/shtick/diag-optimized.txt
|
107 | 0 |
| A |
src/shtick/diag-standard.txt
|
100 | 0 |
| A |
src/shtick/performanc_diagnostic.py
|
184 | 0 |
src/shtick/bench-intensive-opt-20250602-223501.outadded@@ -0,0 +1,437 @@ | ||
| 1 | +SHTICK INTENSIVE PERFORMANCE BENCHMARK RESULTS | |
| 2 | +Version: OPTIMIZED | |
| 3 | +Timestamp: 2025-06-02 22:35:01 | |
| 4 | +================================================================================ | |
| 5 | + | |
| 6 | +PERFORMANCE SUMMARY (all times in milliseconds) | |
| 7 | +-------------------------------------------------------------------------------- | |
| 8 | +Category Operation Mean Items/sec | |
| 9 | +-------------------------------------------------------------------------------- | |
| 10 | +Batch Operations Batch add (10 items) 3.28 3,048 | |
| 11 | +Batch Operations Batch add (100 items) 5.48 18,243 | |
| 12 | +Batch Operations Batch add (50 items) 4.03 12,405 | |
| 13 | +Batch Operations Batch add (500 items) 12.89 38,796 | |
| 14 | +Conflict Checking Check 100 new items 0.14 | |
| 15 | +Conflict Checking Check 50 existing items 0.02 | |
| 16 | +Conflict Checking Repeated 200 checks 0.05 | |
| 17 | +File Generation Generate 10 items 0.52 | |
| 18 | +File Generation Generate 100 items 1.06 | |
| 19 | +File Generation Generate 500 items 3.37 | |
| 20 | +File Generation Incremental 10 items (1 change) 1.45 | |
| 21 | +File Generation Incremental 10 items (15 changes) 1.60 | |
| 22 | +File Generation Incremental 100 items (1 change) 2.77 | |
| 23 | +File Generation Incremental 100 items (15 changes) 2.78 | |
| 24 | +File Generation Incremental 500 items (1 change) 7.50 | |
| 25 | +File Generation Incremental 500 items (15 changes) 7.52 | |
| 26 | +Key Validation Validate 10000 keys 3.07 | |
| 27 | +List Operations List all (2085 items) 7.95 262,402 | |
| 28 | +List Operations List group (250 items) 0.94 267,013 | |
| 29 | +Stress Test 100 conflict checks 0.70 | |
| 30 | +Stress Test Create 9220 items 90802.51 | |
| 31 | +Stress Test List 9220 items 38.16 | |
| 32 | + | |
| 33 | + | |
| 34 | +OPTIMIZATION IMPACT | |
| 35 | +================================================================================ | |
| 36 | +This version includes optimizations: | |
| 37 | +✓ Pre-compiled regex patterns for key validation | |
| 38 | +✓ LRU caching for conflict checks and lookups | |
| 39 | +✓ Incremental file generation | |
| 40 | +✓ Cached settings and shell detection | |
| 41 | + | |
| 42 | + | |
| 43 | +DETAILED RESULTS | |
| 44 | +================================================================================ | |
| 45 | + | |
| 46 | +Batch Operations: | |
| 47 | +---------------- | |
| 48 | + Batch add (10 items): | |
| 49 | + Mean: 3.281ms | |
| 50 | + Median: 3.268ms | |
| 51 | + Min: 3.253ms | |
| 52 | + Max: 3.316ms | |
| 53 | + StdDev: 0.029ms | |
| 54 | + | |
| 55 | + Batch add (100 items): | |
| 56 | + Mean: 5.482ms | |
| 57 | + Median: 5.542ms | |
| 58 | + Min: 5.126ms | |
| 59 | + Max: 5.809ms | |
| 60 | + StdDev: 0.268ms | |
| 61 | + | |
| 62 | + Batch add (50 items): | |
| 63 | + Mean: 4.030ms | |
| 64 | + Median: 4.085ms | |
| 65 | + Min: 3.849ms | |
| 66 | + Max: 4.159ms | |
| 67 | + StdDev: 0.133ms | |
| 68 | + | |
| 69 | + Batch add (500 items): | |
| 70 | + Mean: 12.888ms | |
| 71 | + Median: 13.003ms | |
| 72 | + Min: 11.317ms | |
| 73 | + Max: 14.222ms | |
| 74 | + StdDev: 1.313ms | |
| 75 | + | |
| 76 | + | |
| 77 | +Conflict Checking: | |
| 78 | +----------------- | |
| 79 | + Check 100 new items: | |
| 80 | + Mean: 0.141ms | |
| 81 | + Median: 0.136ms | |
| 82 | + Min: 0.131ms | |
| 83 | + Max: 0.178ms | |
| 84 | + StdDev: 0.014ms | |
| 85 | + | |
| 86 | + Check 50 existing items: | |
| 87 | + Mean: 0.019ms | |
| 88 | + Median: 0.019ms | |
| 89 | + Min: 0.019ms | |
| 90 | + Max: 0.020ms | |
| 91 | + StdDev: 0.000ms | |
| 92 | + | |
| 93 | + Repeated 200 checks: | |
| 94 | + Mean: 0.050ms | |
| 95 | + Median: 0.050ms | |
| 96 | + Min: 0.049ms | |
| 97 | + Max: 0.051ms | |
| 98 | + StdDev: 0.001ms | |
| 99 | + Notes: With LRU cache | |
| 100 | + | |
| 101 | + | |
| 102 | +File Generation: | |
| 103 | +--------------- | |
| 104 | + Generate 10 items: | |
| 105 | + Mean: 0.525ms | |
| 106 | + Median: 0.519ms | |
| 107 | + Min: 0.503ms | |
| 108 | + Max: 0.553ms | |
| 109 | + StdDev: 0.019ms | |
| 110 | + | |
| 111 | + Generate 100 items: | |
| 112 | + Mean: 1.059ms | |
| 113 | + Median: 1.055ms | |
| 114 | + Min: 1.052ms | |
| 115 | + Max: 1.069ms | |
| 116 | + StdDev: 0.008ms | |
| 117 | + | |
| 118 | + Generate 500 items: | |
| 119 | + Mean: 3.367ms | |
| 120 | + Median: 3.370ms | |
| 121 | + Min: 3.336ms | |
| 122 | + Max: 3.386ms | |
| 123 | + StdDev: 0.019ms | |
| 124 | + | |
| 125 | + Incremental 10 items (1 change): | |
| 126 | + Mean: 1.447ms | |
| 127 | + Median: 1.425ms | |
| 128 | + Min: 1.404ms | |
| 129 | + Max: 1.636ms | |
| 130 | + StdDev: 0.070ms | |
| 131 | + | |
| 132 | + Incremental 10 items (15 changes): | |
| 133 | + Mean: 1.603ms | |
| 134 | + Median: 1.484ms | |
| 135 | + Min: 1.446ms | |
| 136 | + Max: 1.956ms | |
| 137 | + StdDev: 0.217ms | |
| 138 | + | |
| 139 | + Incremental 100 items (1 change): | |
| 140 | + Mean: 2.767ms | |
| 141 | + Median: 2.752ms | |
| 142 | + Min: 2.620ms | |
| 143 | + Max: 3.013ms | |
| 144 | + StdDev: 0.122ms | |
| 145 | + | |
| 146 | + Incremental 100 items (15 changes): | |
| 147 | + Mean: 2.780ms | |
| 148 | + Median: 2.702ms | |
| 149 | + Min: 2.626ms | |
| 150 | + Max: 3.187ms | |
| 151 | + StdDev: 0.196ms | |
| 152 | + | |
| 153 | + Incremental 500 items (1 change): | |
| 154 | + Mean: 7.498ms | |
| 155 | + Median: 7.466ms | |
| 156 | + Min: 7.402ms | |
| 157 | + Max: 7.677ms | |
| 158 | + StdDev: 0.095ms | |
| 159 | + | |
| 160 | + Incremental 500 items (15 changes): | |
| 161 | + Mean: 7.518ms | |
| 162 | + Median: 7.511ms | |
| 163 | + Min: 7.454ms | |
| 164 | + Max: 7.593ms | |
| 165 | + StdDev: 0.049ms | |
| 166 | + | |
| 167 | + | |
| 168 | +Key Validation: | |
| 169 | +-------------- | |
| 170 | + Validate 10000 keys: | |
| 171 | + Mean: 3.069ms | |
| 172 | + Median: 3.056ms | |
| 173 | + Min: 3.011ms | |
| 174 | + Max: 3.173ms | |
| 175 | + StdDev: 0.063ms | |
| 176 | + | |
| 177 | + | |
| 178 | +List Operations: | |
| 179 | +--------------- | |
| 180 | + List all (2085 items): | |
| 181 | + Mean: 7.946ms | |
| 182 | + Median: 7.849ms | |
| 183 | + Min: 7.819ms | |
| 184 | + Max: 8.600ms | |
| 185 | + StdDev: 0.230ms | |
| 186 | + | |
| 187 | + List group (250 items): | |
| 188 | + Mean: 0.936ms | |
| 189 | + Median: 0.929ms | |
| 190 | + Min: 0.924ms | |
| 191 | + Max: 1.016ms | |
| 192 | + StdDev: 0.018ms | |
| 193 | + | |
| 194 | + | |
| 195 | +Stress Test: | |
| 196 | +----------- | |
| 197 | + 100 conflict checks: | |
| 198 | + Mean: 0.702ms | |
| 199 | + Median: 0.702ms | |
| 200 | + Min: 0.702ms | |
| 201 | + Max: 0.702ms | |
| 202 | + | |
| 203 | + Create 9220 items: | |
| 204 | + Mean: 90802.510ms | |
| 205 | + Median: 90802.510ms | |
| 206 | + Min: 90802.510ms | |
| 207 | + Max: 90802.510ms | |
| 208 | + | |
| 209 | + List 9220 items: | |
| 210 | + Mean: 38.159ms | |
| 211 | + Median: 38.159ms | |
| 212 | + Min: 38.159ms | |
| 213 | + Max: 38.159ms | |
| 214 | + | |
| 215 | + | |
| 216 | + | |
| 217 | +JSON DATA | |
| 218 | +================================================================================ | |
| 219 | +{ | |
| 220 | + "timestamp": "2025-06-02T22:35:01.255554", | |
| 221 | + "version": { | |
| 222 | + "optimized": true, | |
| 223 | + "version": "optimized" | |
| 224 | + }, | |
| 225 | + "results": { | |
| 226 | + "Key Validation": { | |
| 227 | + "Validate 10000 keys": { | |
| 228 | + "mean_ms": 3.0689167993841693, | |
| 229 | + "median_ms": 3.0559580045519397, | |
| 230 | + "min_ms": 3.0106670019449666, | |
| 231 | + "max_ms": 3.172625001752749, | |
| 232 | + "stdev_ms": 0.0628303224591866, | |
| 233 | + "total_ms": 15.344583996920846, | |
| 234 | + "notes": null | |
| 235 | + } | |
| 236 | + }, | |
| 237 | + "Conflict Checking": { | |
| 238 | + "Check 100 new items": { | |
| 239 | + "mean_ms": 0.14068739692447707, | |
| 240 | + "median_ms": 0.1360414971713908, | |
| 241 | + "min_ms": 0.1306249905610457, | |
| 242 | + "max_ms": 0.17833299352787435, | |
| 243 | + "stdev_ms": 0.014313103476473174, | |
| 244 | + "total_ms": 1.4068739692447707, | |
| 245 | + "notes": null | |
| 246 | + }, | |
| 247 | + "Check 50 existing items": { | |
| 248 | + "mean_ms": 0.01943749957717955, | |
| 249 | + "median_ms": 0.019416504073888063, | |
| 250 | + "min_ms": 0.019041995983570814, | |
| 251 | + "max_ms": 0.02012499317061156, | |
| 252 | + "stdev_ms": 0.00031626079491064703, | |
| 253 | + "total_ms": 0.1943749957717955, | |
| 254 | + "notes": null | |
| 255 | + }, | |
| 256 | + "Repeated 200 checks": { | |
| 257 | + "mean_ms": 0.04999979864805937, | |
| 258 | + "median_ms": 0.05008299194741994, | |
| 259 | + "min_ms": 0.04874999285675585, | |
| 260 | + "max_ms": 0.05095799861010164, | |
| 261 | + "stdev_ms": 0.0009614696328708676, | |
| 262 | + "total_ms": 0.24999899324029684, | |
| 263 | + "notes": "With LRU cache" | |
| 264 | + } | |
| 265 | + }, | |
| 266 | + "List Operations": { | |
| 267 | + "List all (2085 items)": { | |
| 268 | + "mean_ms": 7.945835350255948, | |
| 269 | + "median_ms": 7.84912500239443, | |
| 270 | + "min_ms": 7.818583006155677, | |
| 271 | + "max_ms": 8.59962499816902, | |
| 272 | + "stdev_ms": 0.2298286627253673, | |
| 273 | + "total_ms": 158.91670700511895, | |
| 274 | + "notes": null | |
| 275 | + }, | |
| 276 | + "List group (250 items)": { | |
| 277 | + "mean_ms": 0.9362847316272868, | |
| 278 | + "median_ms": 0.9289169975090772, | |
| 279 | + "min_ms": 0.9244999964721501, | |
| 280 | + "max_ms": 1.0164169943891466, | |
| 281 | + "stdev_ms": 0.018272351055007935, | |
| 282 | + "total_ms": 28.0885419488186, | |
| 283 | + "notes": null | |
| 284 | + } | |
| 285 | + }, | |
| 286 | + "Batch Operations": { | |
| 287 | + "Batch add (10 items)": { | |
| 288 | + "mean_ms": 3.2811000011861324, | |
| 289 | + "median_ms": 3.268459011451341, | |
| 290 | + "min_ms": 3.252958005759865, | |
| 291 | + "max_ms": 3.3157500001834705, | |
| 292 | + "stdev_ms": 0.029424444786398975, | |
| 293 | + "total_ms": 16.405500005930662, | |
| 294 | + "notes": null | |
| 295 | + }, | |
| 296 | + "Batch add (50 items)": { | |
| 297 | + "mean_ms": 4.030483600217849, | |
| 298 | + "median_ms": 4.085459004272707, | |
| 299 | + "min_ms": 3.8490839942824095, | |
| 300 | + "max_ms": 4.158708004979417, | |
| 301 | + "stdev_ms": 0.13261041492843492, | |
| 302 | + "total_ms": 20.152418001089245, | |
| 303 | + "notes": null | |
| 304 | + }, | |
| 305 | + "Batch add (100 items)": { | |
| 306 | + "mean_ms": 5.481574995792471, | |
| 307 | + "median_ms": 5.541749997064471, | |
| 308 | + "min_ms": 5.125707990373485, | |
| 309 | + "max_ms": 5.808834001072682, | |
| 310 | + "stdev_ms": 0.26840105575350065, | |
| 311 | + "total_ms": 27.407874978962354, | |
| 312 | + "notes": null | |
| 313 | + }, | |
| 314 | + "Batch add (500 items)": { | |
| 315 | + "mean_ms": 12.888050003675744, | |
| 316 | + "median_ms": 13.002833002246916, | |
| 317 | + "min_ms": 11.316834003082477, | |
| 318 | + "max_ms": 14.22179200744722, | |
| 319 | + "stdev_ms": 1.3130454879786941, | |
| 320 | + "total_ms": 64.44025001837872, | |
| 321 | + "notes": null | |
| 322 | + } | |
| 323 | + }, | |
| 324 | + "File Generation": { | |
| 325 | + "Generate 10 items": { | |
| 326 | + "mean_ms": 0.5247916036751121, | |
| 327 | + "median_ms": 0.5186670023249462, | |
| 328 | + "min_ms": 0.5029159947298467, | |
| 329 | + "max_ms": 0.5527080065803602, | |
| 330 | + "stdev_ms": 0.018935922427035234, | |
| 331 | + "total_ms": 2.6239580183755606, | |
| 332 | + "notes": null | |
| 333 | + }, | |
| 334 | + "Incremental 10 items (1 change)": { | |
| 335 | + "mean_ms": 1.4471749993390404, | |
| 336 | + "median_ms": 1.4251044922275469, | |
| 337 | + "min_ms": 1.4037080109119415, | |
| 338 | + "max_ms": 1.6364590119337663, | |
| 339 | + "stdev_ms": 0.06966702276462376, | |
| 340 | + "total_ms": 14.471749993390404, | |
| 341 | + "notes": null | |
| 342 | + }, | |
| 343 | + "Incremental 10 items (15 changes)": { | |
| 344 | + "mean_ms": 1.6028916026698425, | |
| 345 | + "median_ms": 1.4838540009805001, | |
| 346 | + "min_ms": 1.4459999947575852, | |
| 347 | + "max_ms": 1.9561660010367632, | |
| 348 | + "stdev_ms": 0.216529302444539, | |
| 349 | + "total_ms": 16.028916026698425, | |
| 350 | + "notes": null | |
| 351 | + }, | |
| 352 | + "Generate 100 items": { | |
| 353 | + "mean_ms": 1.0585084004560485, | |
| 354 | + "median_ms": 1.0547090059844777, | |
| 355 | + "min_ms": 1.051707993610762, | |
| 356 | + "max_ms": 1.0685000015655532, | |
| 357 | + "stdev_ms": 0.007718667723822919, | |
| 358 | + "total_ms": 5.292542002280243, | |
| 359 | + "notes": null | |
| 360 | + }, | |
| 361 | + "Incremental 100 items (1 change)": { | |
| 362 | + "mean_ms": 2.7665791974868625, | |
| 363 | + "median_ms": 2.752457992755808, | |
| 364 | + "min_ms": 2.620250001200475, | |
| 365 | + "max_ms": 3.0131669918773696, | |
| 366 | + "stdev_ms": 0.1221665675248218, | |
| 367 | + "total_ms": 27.665791974868625, | |
| 368 | + "notes": null | |
| 369 | + }, | |
| 370 | + "Incremental 100 items (15 changes)": { | |
| 371 | + "mean_ms": 2.7801917007309385, | |
| 372 | + "median_ms": 2.701541998249013, | |
| 373 | + "min_ms": 2.626375004183501, | |
| 374 | + "max_ms": 3.1865409982856363, | |
| 375 | + "stdev_ms": 0.19603674872801233, | |
| 376 | + "total_ms": 27.801917007309385, | |
| 377 | + "notes": null | |
| 378 | + }, | |
| 379 | + "Generate 500 items": { | |
| 380 | + "mean_ms": 3.3670585980871692, | |
| 381 | + "median_ms": 3.3700420026434585, | |
| 382 | + "min_ms": 3.335874993354082, | |
| 383 | + "max_ms": 3.3855829969979823, | |
| 384 | + "stdev_ms": 0.018825527910995402, | |
| 385 | + "total_ms": 16.835292990435846, | |
| 386 | + "notes": null | |
| 387 | + }, | |
| 388 | + "Incremental 500 items (1 change)": { | |
| 389 | + "mean_ms": 7.498041600047145, | |
| 390 | + "median_ms": 7.466208502592053, | |
| 391 | + "min_ms": 7.402292001643218, | |
| 392 | + "max_ms": 7.67700000142213, | |
| 393 | + "stdev_ms": 0.09482651625741934, | |
| 394 | + "total_ms": 74.98041600047145, | |
| 395 | + "notes": null | |
| 396 | + }, | |
| 397 | + "Incremental 500 items (15 changes)": { | |
| 398 | + "mean_ms": 7.518220799101982, | |
| 399 | + "median_ms": 7.510957999329548, | |
| 400 | + "min_ms": 7.453875005012378, | |
| 401 | + "max_ms": 7.593207992613316, | |
| 402 | + "stdev_ms": 0.04879667079636522, | |
| 403 | + "total_ms": 75.18220799101982, | |
| 404 | + "notes": null | |
| 405 | + } | |
| 406 | + }, | |
| 407 | + "Stress Test": { | |
| 408 | + "Create 9220 items": { | |
| 409 | + "mean_ms": 90802.50991698995, | |
| 410 | + "median_ms": 90802.50991698995, | |
| 411 | + "min_ms": 90802.50991698995, | |
| 412 | + "max_ms": 90802.50991698995, | |
| 413 | + "stdev_ms": 0, | |
| 414 | + "total_ms": 0, | |
| 415 | + "notes": null | |
| 416 | + }, | |
| 417 | + "List 9220 items": { | |
| 418 | + "mean_ms": 38.15937500621658, | |
| 419 | + "median_ms": 38.15937500621658, | |
| 420 | + "min_ms": 38.15937500621658, | |
| 421 | + "max_ms": 38.15937500621658, | |
| 422 | + "stdev_ms": 0, | |
| 423 | + "total_ms": 0, | |
| 424 | + "notes": null | |
| 425 | + }, | |
| 426 | + "100 conflict checks": { | |
| 427 | + "mean_ms": 0.7022499921731651, | |
| 428 | + "median_ms": 0.7022499921731651, | |
| 429 | + "min_ms": 0.7022499921731651, | |
| 430 | + "max_ms": 0.7022499921731651, | |
| 431 | + "stdev_ms": 0, | |
| 432 | + "total_ms": 0, | |
| 433 | + "notes": null | |
| 434 | + } | |
| 435 | + } | |
| 436 | + } | |
| 437 | +} | |
src/shtick/bench-intensive-std-20250602-223632.outadded@@ -0,0 +1,310 @@ | ||
| 1 | +SHTICK INTENSIVE PERFORMANCE BENCHMARK RESULTS | |
| 2 | +Version: STANDARD | |
| 3 | +Timestamp: 2025-06-02 22:36:32 | |
| 4 | +================================================================================ | |
| 5 | + | |
| 6 | +PERFORMANCE SUMMARY (all times in milliseconds) | |
| 7 | +-------------------------------------------------------------------------------- | |
| 8 | +Category Operation Mean Items/sec | |
| 9 | +-------------------------------------------------------------------------------- | |
| 10 | +Batch Operations Batch add (10 items) 0.39 25,535 | |
| 11 | +Batch Operations Batch add (100 items) 1.24 80,535 | |
| 12 | +Batch Operations Batch add (50 items) 0.79 63,479 | |
| 13 | +Batch Operations Batch add (500 items) 4.61 108,525 | |
| 14 | +Conflict Checking Check 100 new items 0.20 | |
| 15 | +Conflict Checking Check 50 existing items 0.06 | |
| 16 | +Conflict Checking Repeated 200 checks 0.22 | |
| 17 | +File Generation Generate 10 items 0.37 | |
| 18 | +File Generation Generate 100 items 0.66 | |
| 19 | +File Generation Generate 500 items 1.63 | |
| 20 | +List Operations List all (2085 items) 7.89 264,109 | |
| 21 | +List Operations List group (250 items) 0.96 259,960 | |
| 22 | +Stress Test 100 conflict checks 0.59 | |
| 23 | +Stress Test Create 9220 items 30126.58 | |
| 24 | +Stress Test List 9220 items 43.68 | |
| 25 | + | |
| 26 | + | |
| 27 | +OPTIMIZATION IMPACT | |
| 28 | +================================================================================ | |
| 29 | +This is the standard version without optimizations. | |
| 30 | +Compare with optimized version to see improvements. | |
| 31 | + | |
| 32 | + | |
| 33 | +DETAILED RESULTS | |
| 34 | +================================================================================ | |
| 35 | + | |
| 36 | +Batch Operations: | |
| 37 | +---------------- | |
| 38 | + Batch add (10 items): | |
| 39 | + Mean: 0.392ms | |
| 40 | + Median: 0.392ms | |
| 41 | + Min: 0.381ms | |
| 42 | + Max: 0.411ms | |
| 43 | + StdDev: 0.012ms | |
| 44 | + | |
| 45 | + Batch add (100 items): | |
| 46 | + Mean: 1.242ms | |
| 47 | + Median: 1.185ms | |
| 48 | + Min: 1.106ms | |
| 49 | + Max: 1.583ms | |
| 50 | + StdDev: 0.193ms | |
| 51 | + | |
| 52 | + Batch add (50 items): | |
| 53 | + Mean: 0.788ms | |
| 54 | + Median: 0.718ms | |
| 55 | + Min: 0.692ms | |
| 56 | + Max: 1.081ms | |
| 57 | + StdDev: 0.165ms | |
| 58 | + | |
| 59 | + Batch add (500 items): | |
| 60 | + Mean: 4.607ms | |
| 61 | + Median: 4.424ms | |
| 62 | + Min: 4.196ms | |
| 63 | + Max: 5.317ms | |
| 64 | + StdDev: 0.432ms | |
| 65 | + | |
| 66 | + | |
| 67 | +Conflict Checking: | |
| 68 | +----------------- | |
| 69 | + Check 100 new items: | |
| 70 | + Mean: 0.200ms | |
| 71 | + Median: 0.200ms | |
| 72 | + Min: 0.199ms | |
| 73 | + Max: 0.203ms | |
| 74 | + StdDev: 0.001ms | |
| 75 | + | |
| 76 | + Check 50 existing items: | |
| 77 | + Mean: 0.063ms | |
| 78 | + Median: 0.063ms | |
| 79 | + Min: 0.059ms | |
| 80 | + Max: 0.068ms | |
| 81 | + StdDev: 0.002ms | |
| 82 | + | |
| 83 | + Repeated 200 checks: | |
| 84 | + Mean: 0.224ms | |
| 85 | + Median: 0.225ms | |
| 86 | + Min: 0.211ms | |
| 87 | + Max: 0.234ms | |
| 88 | + StdDev: 0.008ms | |
| 89 | + Notes: No cache | |
| 90 | + | |
| 91 | + | |
| 92 | +File Generation: | |
| 93 | +--------------- | |
| 94 | + Generate 10 items: | |
| 95 | + Mean: 0.366ms | |
| 96 | + Median: 0.363ms | |
| 97 | + Min: 0.362ms | |
| 98 | + Max: 0.377ms | |
| 99 | + StdDev: 0.006ms | |
| 100 | + | |
| 101 | + Generate 100 items: | |
| 102 | + Mean: 0.658ms | |
| 103 | + Median: 0.658ms | |
| 104 | + Min: 0.648ms | |
| 105 | + Max: 0.666ms | |
| 106 | + StdDev: 0.007ms | |
| 107 | + | |
| 108 | + Generate 500 items: | |
| 109 | + Mean: 1.626ms | |
| 110 | + Median: 1.569ms | |
| 111 | + Min: 1.564ms | |
| 112 | + Max: 1.834ms | |
| 113 | + StdDev: 0.117ms | |
| 114 | + | |
| 115 | + | |
| 116 | +List Operations: | |
| 117 | +--------------- | |
| 118 | + List all (2085 items): | |
| 119 | + Mean: 7.894ms | |
| 120 | + Median: 7.894ms | |
| 121 | + Min: 7.844ms | |
| 122 | + Max: 7.976ms | |
| 123 | + StdDev: 0.027ms | |
| 124 | + | |
| 125 | + List group (250 items): | |
| 126 | + Mean: 0.962ms | |
| 127 | + Median: 0.942ms | |
| 128 | + Min: 0.916ms | |
| 129 | + Max: 1.155ms | |
| 130 | + StdDev: 0.053ms | |
| 131 | + | |
| 132 | + | |
| 133 | +Stress Test: | |
| 134 | +----------- | |
| 135 | + 100 conflict checks: | |
| 136 | + Mean: 0.588ms | |
| 137 | + Median: 0.588ms | |
| 138 | + Min: 0.588ms | |
| 139 | + Max: 0.588ms | |
| 140 | + | |
| 141 | + Create 9220 items: | |
| 142 | + Mean: 30126.585ms | |
| 143 | + Median: 30126.585ms | |
| 144 | + Min: 30126.585ms | |
| 145 | + Max: 30126.585ms | |
| 146 | + | |
| 147 | + List 9220 items: | |
| 148 | + Mean: 43.679ms | |
| 149 | + Median: 43.679ms | |
| 150 | + Min: 43.679ms | |
| 151 | + Max: 43.679ms | |
| 152 | + | |
| 153 | + | |
| 154 | + | |
| 155 | +JSON DATA | |
| 156 | +================================================================================ | |
| 157 | +{ | |
| 158 | + "timestamp": "2025-06-02T22:36:32.468423", | |
| 159 | + "version": { | |
| 160 | + "optimized": false, | |
| 161 | + "version": "standard" | |
| 162 | + }, | |
| 163 | + "results": { | |
| 164 | + "Conflict Checking": { | |
| 165 | + "Check 100 new items": { | |
| 166 | + "mean_ms": 0.20017069909954444, | |
| 167 | + "median_ms": 0.19999950018245727, | |
| 168 | + "min_ms": 0.1986250135814771, | |
| 169 | + "max_ms": 0.20287500228732824, | |
| 170 | + "stdev_ms": 0.001198303071867976, | |
| 171 | + "total_ms": 2.0017069909954444, | |
| 172 | + "notes": null | |
| 173 | + }, | |
| 174 | + "Check 50 existing items": { | |
| 175 | + "mean_ms": 0.06296659994404763, | |
| 176 | + "median_ms": 0.06306199793471023, | |
| 177 | + "min_ms": 0.05908399180043489, | |
| 178 | + "max_ms": 0.06754200148861855, | |
| 179 | + "stdev_ms": 0.002397014817112448, | |
| 180 | + "total_ms": 0.6296659994404763, | |
| 181 | + "notes": null | |
| 182 | + }, | |
| 183 | + "Repeated 200 checks": { | |
| 184 | + "mean_ms": 0.22413360129576176, | |
| 185 | + "median_ms": 0.22483400243800133, | |
| 186 | + "min_ms": 0.21091700182296336, | |
| 187 | + "max_ms": 0.23350000265054405, | |
| 188 | + "stdev_ms": 0.008347898035451077, | |
| 189 | + "total_ms": 1.1206680064788088, | |
| 190 | + "notes": "No cache" | |
| 191 | + } | |
| 192 | + }, | |
| 193 | + "List Operations": { | |
| 194 | + "List all (2085 items)": { | |
| 195 | + "mean_ms": 7.894462651893264, | |
| 196 | + "median_ms": 7.8942085019662045, | |
| 197 | + "min_ms": 7.843583007343113, | |
| 198 | + "max_ms": 7.97624999540858, | |
| 199 | + "stdev_ms": 0.02673801806206058, | |
| 200 | + "total_ms": 157.88925303786527, | |
| 201 | + "notes": null | |
| 202 | + }, | |
| 203 | + "List group (250 items)": { | |
| 204 | + "mean_ms": 0.9616861333294461, | |
| 205 | + "median_ms": 0.9419584966963157, | |
| 206 | + "min_ms": 0.9160000045085326, | |
| 207 | + "max_ms": 1.154749988927506, | |
| 208 | + "stdev_ms": 0.05308687229361325, | |
| 209 | + "total_ms": 28.850583999883384, | |
| 210 | + "notes": null | |
| 211 | + } | |
| 212 | + }, | |
| 213 | + "Batch Operations": { | |
| 214 | + "Batch add (10 items)": { | |
| 215 | + "mean_ms": 0.3916165995178744, | |
| 216 | + "median_ms": 0.3918750007869676, | |
| 217 | + "min_ms": 0.38058299105614424, | |
| 218 | + "max_ms": 0.4108749999431893, | |
| 219 | + "stdev_ms": 0.012071888040208655, | |
| 220 | + "total_ms": 1.958082997589372, | |
| 221 | + "notes": null | |
| 222 | + }, | |
| 223 | + "Batch add (50 items)": { | |
| 224 | + "mean_ms": 0.7876666000811383, | |
| 225 | + "median_ms": 0.7182910048868507, | |
| 226 | + "min_ms": 0.6916669954080135, | |
| 227 | + "max_ms": 1.0805000056279823, | |
| 228 | + "stdev_ms": 0.1651712699869281, | |
| 229 | + "total_ms": 3.9383330004056916, | |
| 230 | + "notes": null | |
| 231 | + }, | |
| 232 | + "Batch add (100 items)": { | |
| 233 | + "mean_ms": 1.2417002028087154, | |
| 234 | + "median_ms": 1.1849159927805886, | |
| 235 | + "min_ms": 1.1060840042773634, | |
| 236 | + "max_ms": 1.5825840091565624, | |
| 237 | + "stdev_ms": 0.19347071555544915, | |
| 238 | + "total_ms": 6.208501014043577, | |
| 239 | + "notes": null | |
| 240 | + }, | |
| 241 | + "Batch add (500 items)": { | |
| 242 | + "mean_ms": 4.607216801377945, | |
| 243 | + "median_ms": 4.423749996931292, | |
| 244 | + "min_ms": 4.19579200388398, | |
| 245 | + "max_ms": 5.317332994309254, | |
| 246 | + "stdev_ms": 0.4317639295676596, | |
| 247 | + "total_ms": 23.036084006889723, | |
| 248 | + "notes": null | |
| 249 | + } | |
| 250 | + }, | |
| 251 | + "File Generation": { | |
| 252 | + "Generate 10 items": { | |
| 253 | + "mean_ms": 0.365991800208576, | |
| 254 | + "median_ms": 0.36345800617709756, | |
| 255 | + "min_ms": 0.3617089969338849, | |
| 256 | + "max_ms": 0.37733299541287124, | |
| 257 | + "stdev_ms": 0.00642998838481809, | |
| 258 | + "total_ms": 1.8299590010428801, | |
| 259 | + "notes": null | |
| 260 | + }, | |
| 261 | + "Generate 100 items": { | |
| 262 | + "mean_ms": 0.6577169959200546, | |
| 263 | + "median_ms": 0.6584169896086678, | |
| 264 | + "min_ms": 0.6480839947471395, | |
| 265 | + "max_ms": 0.6663340027444065, | |
| 266 | + "stdev_ms": 0.006563855201346565, | |
| 267 | + "total_ms": 3.288584979600273, | |
| 268 | + "notes": null | |
| 269 | + }, | |
| 270 | + "Generate 500 items": { | |
| 271 | + "mean_ms": 1.6263416007859632, | |
| 272 | + "median_ms": 1.5686249971622601, | |
| 273 | + "min_ms": 1.5641250065527856, | |
| 274 | + "max_ms": 1.8341660033911467, | |
| 275 | + "stdev_ms": 0.1168928398687913, | |
| 276 | + "total_ms": 8.131708003929816, | |
| 277 | + "notes": null | |
| 278 | + } | |
| 279 | + }, | |
| 280 | + "Stress Test": { | |
| 281 | + "Create 9220 items": { | |
| 282 | + "mean_ms": 30126.584999990882, | |
| 283 | + "median_ms": 30126.584999990882, | |
| 284 | + "min_ms": 30126.584999990882, | |
| 285 | + "max_ms": 30126.584999990882, | |
| 286 | + "stdev_ms": 0, | |
| 287 | + "total_ms": 0, | |
| 288 | + "notes": null | |
| 289 | + }, | |
| 290 | + "List 9220 items": { | |
| 291 | + "mean_ms": 43.678833011654206, | |
| 292 | + "median_ms": 43.678833011654206, | |
| 293 | + "min_ms": 43.678833011654206, | |
| 294 | + "max_ms": 43.678833011654206, | |
| 295 | + "stdev_ms": 0, | |
| 296 | + "total_ms": 0, | |
| 297 | + "notes": null | |
| 298 | + }, | |
| 299 | + "100 conflict checks": { | |
| 300 | + "mean_ms": 0.587959002587013, | |
| 301 | + "median_ms": 0.587959002587013, | |
| 302 | + "min_ms": 0.587959002587013, | |
| 303 | + "max_ms": 0.587959002587013, | |
| 304 | + "stdev_ms": 0, | |
| 305 | + "total_ms": 0, | |
| 306 | + "notes": null | |
| 307 | + } | |
| 308 | + } | |
| 309 | + } | |
| 310 | +} | |
src/shtick/bench.pyadded@@ -0,0 +1,724 @@ | ||
| 1 | +#!/usr/bin/env python3 | |
| 2 | +""" | |
| 3 | +Intensive shtick performance benchmark | |
| 4 | +Tests with larger datasets to show real performance differences | |
| 5 | +""" | |
| 6 | + | |
| 7 | +import time | |
| 8 | +import tempfile | |
| 9 | +import os | |
| 10 | +import sys | |
| 11 | +import shutil | |
| 12 | +from pathlib import Path | |
| 13 | +import statistics | |
| 14 | +from datetime import datetime | |
| 15 | +import json | |
| 16 | +import random | |
| 17 | +import string | |
| 18 | + | |
| 19 | +# Fix import paths | |
| 20 | +current_file = Path(__file__).resolve() | |
| 21 | +shtick_dir = current_file.parent | |
| 22 | +src_dir = shtick_dir.parent | |
| 23 | +project_root = src_dir.parent | |
| 24 | + | |
| 25 | +sys.path.insert(0, str(shtick_dir)) | |
| 26 | +sys.path.insert(0, str(src_dir)) | |
| 27 | +sys.path.insert(0, str(project_root)) | |
| 28 | + | |
| 29 | +# Import handling | |
| 30 | +SHTICK_AVAILABLE = False | |
| 31 | +OPTIMIZED_VERSION = False | |
| 32 | + | |
| 33 | +try: | |
| 34 | + from shtick import ShtickManager | |
| 35 | + from shtick.config import Config | |
| 36 | + from shtick.generator import Generator | |
| 37 | + | |
| 38 | + SHTICK_AVAILABLE = True | |
| 39 | + print("✓ Successfully imported shtick modules") | |
| 40 | + | |
| 41 | + if hasattr(ShtickManager, "_get_all_items_by_type"): | |
| 42 | + OPTIMIZED_VERSION = True | |
| 43 | + print("✓ Detected OPTIMIZED version with caching") | |
| 44 | + else: | |
| 45 | + print("✓ Detected STANDARD version without optimizations") | |
| 46 | + | |
| 47 | +except ImportError: | |
| 48 | + try: | |
| 49 | + import shtick | |
| 50 | + from config import Config | |
| 51 | + from generator import Generator | |
| 52 | + | |
| 53 | + class ShtickManager(shtick.ShtickManager): | |
| 54 | + pass | |
| 55 | + | |
| 56 | + SHTICK_AVAILABLE = True | |
| 57 | + print("✓ Successfully imported shtick modules directly") | |
| 58 | + | |
| 59 | + except ImportError as e: | |
| 60 | + print(f"✗ Could not import shtick modules: {e}") | |
| 61 | + sys.exit(1) | |
| 62 | + | |
| 63 | + | |
| 64 | +def time_operation(func, iterations=10, warmup=2): | |
| 65 | + """Time an operation with proper warmup""" | |
| 66 | + # Warmup | |
| 67 | + for _ in range(warmup): | |
| 68 | + try: | |
| 69 | + func() | |
| 70 | + except: | |
| 71 | + pass | |
| 72 | + | |
| 73 | + times = [] | |
| 74 | + for _ in range(iterations): | |
| 75 | + start = time.perf_counter() | |
| 76 | + func() | |
| 77 | + end = time.perf_counter() | |
| 78 | + times.append((end - start) * 1000) # ms | |
| 79 | + | |
| 80 | + return { | |
| 81 | + "mean": statistics.mean(times), | |
| 82 | + "median": statistics.median(times), | |
| 83 | + "min": min(times), | |
| 84 | + "max": max(times), | |
| 85 | + "stdev": statistics.stdev(times) if len(times) > 1 else 0, | |
| 86 | + "total": sum(times), | |
| 87 | + } | |
| 88 | + | |
| 89 | + | |
| 90 | +def generate_random_key(prefix="key", length=8): | |
| 91 | + """Generate random key for testing""" | |
| 92 | + suffix = "".join(random.choices(string.ascii_lowercase + string.digits, k=length)) | |
| 93 | + return f"{prefix}_{suffix}" | |
| 94 | + | |
| 95 | + | |
| 96 | +class IntensiveBenchmark: | |
| 97 | + """Intensive benchmark suite with larger datasets""" | |
| 98 | + | |
| 99 | + def __init__(self): | |
| 100 | + self.temp_dir = None | |
| 101 | + self.config_path = None | |
| 102 | + self.manager = None | |
| 103 | + self.results = {} | |
| 104 | + self.version_info = { | |
| 105 | + "optimized": OPTIMIZED_VERSION, | |
| 106 | + "version": "optimized" if OPTIMIZED_VERSION else "standard", | |
| 107 | + } | |
| 108 | + | |
| 109 | + def record_result(self, category, operation, stats, notes=None): | |
| 110 | + """Record benchmark results""" | |
| 111 | + if category not in self.results: | |
| 112 | + self.results[category] = {} | |
| 113 | + self.results[category][operation] = { | |
| 114 | + "mean_ms": stats["mean"], | |
| 115 | + "median_ms": stats["median"], | |
| 116 | + "min_ms": stats["min"], | |
| 117 | + "max_ms": stats["max"], | |
| 118 | + "stdev_ms": stats["stdev"], | |
| 119 | + "total_ms": stats.get("total", 0), | |
| 120 | + "notes": notes, | |
| 121 | + } | |
| 122 | + | |
| 123 | + def setup(self): | |
| 124 | + """Setup test environment""" | |
| 125 | + self.temp_dir = tempfile.mkdtemp(prefix="shtick_bench_") | |
| 126 | + self.config_path = os.path.join(self.temp_dir, "config.toml") | |
| 127 | + | |
| 128 | + with open(self.config_path, "w") as f: | |
| 129 | + f.write("[persistent]\n") | |
| 130 | + | |
| 131 | + # Create manager with debug=False to disable logging | |
| 132 | + self.manager = ShtickManager(config_path=self.config_path, debug=False) | |
| 133 | + | |
| 134 | + # Disable all logging during benchmarks | |
| 135 | + import logging | |
| 136 | + | |
| 137 | + logging.disable(logging.CRITICAL) | |
| 138 | + | |
| 139 | + # Disable auto-generation | |
| 140 | + if hasattr(self.manager, "_save_and_regenerate"): | |
| 141 | + original_save = self.manager._save_and_regenerate | |
| 142 | + | |
| 143 | + def minimal_save(affected_groups=None, changed_items=None): | |
| 144 | + config = self.manager._get_config() | |
| 145 | + config.save() | |
| 146 | + if hasattr(self.manager, "_clear_caches"): | |
| 147 | + self.manager._clear_caches() | |
| 148 | + | |
| 149 | + self.manager._original_save_and_regenerate = original_save | |
| 150 | + self.manager._save_and_regenerate = minimal_save | |
| 151 | + | |
| 152 | + if hasattr(Config, "clear_all_caches"): | |
| 153 | + Config.clear_all_caches() | |
| 154 | + | |
| 155 | + print(f"✓ Test environment created (logging disabled)") | |
| 156 | + | |
| 157 | + def cleanup(self): | |
| 158 | + """Cleanup test environment""" | |
| 159 | + if self.temp_dir and os.path.exists(self.temp_dir): | |
| 160 | + shutil.rmtree(self.temp_dir) | |
| 161 | + | |
| 162 | + shtick_dir = os.path.expanduser("~/.config/shtick") | |
| 163 | + # Clean up all test groups | |
| 164 | + for prefix in ["bench_", "stress_", "large_"]: | |
| 165 | + for entry in os.listdir(shtick_dir) if os.path.exists(shtick_dir) else []: | |
| 166 | + if entry.startswith(prefix): | |
| 167 | + path = os.path.join(shtick_dir, entry) | |
| 168 | + if os.path.isdir(path): | |
| 169 | + shutil.rmtree(path) | |
| 170 | + | |
| 171 | + def benchmark_key_validation_intensive(self): | |
| 172 | + """Test key validation with many keys""" | |
| 173 | + print("\n=== INTENSIVE Key Validation (10,000 keys) ===") | |
| 174 | + | |
| 175 | + # Generate test keys | |
| 176 | + valid_keys = [generate_random_key("valid", 12) for _ in range(5000)] | |
| 177 | + | |
| 178 | + # Generate various invalid keys | |
| 179 | + invalid_keys = [] | |
| 180 | + for i in range(1000): | |
| 181 | + invalid_keys.extend( | |
| 182 | + [ | |
| 183 | + f"{i}invalid", # starts with number | |
| 184 | + f"invalid-{i}!", # invalid character | |
| 185 | + f"inv alid{i}", # space | |
| 186 | + f"-invalid{i}", # starts with dash | |
| 187 | + f"inv@lid{i}", # invalid character | |
| 188 | + ] | |
| 189 | + ) | |
| 190 | + | |
| 191 | + all_keys = valid_keys + invalid_keys | |
| 192 | + random.shuffle(all_keys) | |
| 193 | + | |
| 194 | + # Import validate_key if available | |
| 195 | + try: | |
| 196 | + from shtick.commands import validate_key | |
| 197 | + except: | |
| 198 | + try: | |
| 199 | + from commands import validate_key | |
| 200 | + except: | |
| 201 | + print("Could not import validate_key function") | |
| 202 | + return | |
| 203 | + | |
| 204 | + def validate_all(): | |
| 205 | + valid_count = 0 | |
| 206 | + for key in all_keys: | |
| 207 | + try: | |
| 208 | + validate_key(key) | |
| 209 | + valid_count += 1 | |
| 210 | + except ValueError: | |
| 211 | + pass | |
| 212 | + return valid_count | |
| 213 | + | |
| 214 | + print(f"Validating {len(all_keys)} keys...") | |
| 215 | + stats = time_operation(validate_all, iterations=5) | |
| 216 | + print(f"Total time for {len(all_keys)} validations: {stats['total']:.1f}ms") | |
| 217 | + print(f"Average per 1000 keys: {stats['mean']/10:.2f}ms") | |
| 218 | + print(f"Per-key validation: {stats['mean']/len(all_keys)*1000:.3f}μs") | |
| 219 | + | |
| 220 | + self.record_result("Key Validation", f"Validate {len(all_keys)} keys", stats) | |
| 221 | + | |
| 222 | + def benchmark_conflict_checking_intensive(self): | |
| 223 | + """Test conflict checking with large dataset""" | |
| 224 | + print("\n=== INTENSIVE Conflict Checking ===") | |
| 225 | + | |
| 226 | + # Create a large dataset across multiple groups | |
| 227 | + print("Creating large dataset...") | |
| 228 | + groups = ["work", "dev", "test", "prod", "staging"] | |
| 229 | + items_per_group = 200 | |
| 230 | + | |
| 231 | + start = time.perf_counter() | |
| 232 | + for group in groups: | |
| 233 | + for i in range(items_per_group): | |
| 234 | + self.manager.add_alias( | |
| 235 | + f"{group}_alias_{i}", f"echo {group}_{i}", f"stress_{group}" | |
| 236 | + ) | |
| 237 | + if i % 3 == 0: | |
| 238 | + self.manager.add_env( | |
| 239 | + f"{group}_var_{i}", f"value_{i}", f"stress_{group}" | |
| 240 | + ) | |
| 241 | + setup_time = (time.perf_counter() - start) * 1000 | |
| 242 | + | |
| 243 | + total_items = len(groups) * items_per_group * 4 // 3 # aliases + 1/3 env vars | |
| 244 | + print(f"Created {total_items} items in {setup_time:.0f}ms") | |
| 245 | + | |
| 246 | + # Test 1: Check for non-existent items (no conflicts) | |
| 247 | + print("\nTesting conflict checks for new items...") | |
| 248 | + check_counter = 0 | |
| 249 | + | |
| 250 | + def check_new_items(): | |
| 251 | + nonlocal check_counter | |
| 252 | + for _ in range(100): | |
| 253 | + key = generate_random_key("newitem", 10) | |
| 254 | + self.manager.check_conflicts("alias", key, "stress_work") | |
| 255 | + check_counter += 1 | |
| 256 | + | |
| 257 | + stats = time_operation(check_new_items, iterations=10) | |
| 258 | + print( | |
| 259 | + f"Check 100 new items: {stats['mean']:.2f}ms total, {stats['mean']/100:.3f}ms per check" | |
| 260 | + ) | |
| 261 | + self.record_result("Conflict Checking", "Check 100 new items", stats) | |
| 262 | + | |
| 263 | + # Test 2: Check for existing items (with conflicts) | |
| 264 | + print("\nTesting conflict checks for existing items...") | |
| 265 | + | |
| 266 | + def check_existing_items(): | |
| 267 | + for group in groups: | |
| 268 | + for i in range(0, 100, 10): # Check every 10th item | |
| 269 | + self.manager.check_conflicts( | |
| 270 | + "alias", f"{group}_alias_{i}", "new_group" | |
| 271 | + ) | |
| 272 | + | |
| 273 | + stats = time_operation(check_existing_items, iterations=10) | |
| 274 | + checks = len(groups) * 10 | |
| 275 | + print( | |
| 276 | + f"Check {checks} existing items: {stats['mean']:.2f}ms total, {stats['mean']/checks:.3f}ms per check" | |
| 277 | + ) | |
| 278 | + self.record_result("Conflict Checking", f"Check {checks} existing items", stats) | |
| 279 | + | |
| 280 | + # Test 3: Repeated checks (cache effectiveness) | |
| 281 | + print("\nTesting repeated conflict checks (cache test)...") | |
| 282 | + test_keys = [f"work_alias_{i}" for i in range(20)] | |
| 283 | + | |
| 284 | + def check_repeated(): | |
| 285 | + for _ in range(10): # Repeat 10 times | |
| 286 | + for key in test_keys: | |
| 287 | + self.manager.check_conflicts("alias", key, "test_group") | |
| 288 | + | |
| 289 | + # Clear caches if available | |
| 290 | + if hasattr(self.manager, "_clear_caches"): | |
| 291 | + self.manager._clear_caches() | |
| 292 | + | |
| 293 | + stats = time_operation(check_repeated, iterations=5) | |
| 294 | + total_checks = 10 * len(test_keys) | |
| 295 | + print(f"Repeated checks ({total_checks} total): {stats['mean']:.2f}ms") | |
| 296 | + print(f"Per-check (with potential caching): {stats['mean']/total_checks:.3f}ms") | |
| 297 | + | |
| 298 | + cache_note = "With LRU cache" if OPTIMIZED_VERSION else "No cache" | |
| 299 | + self.record_result( | |
| 300 | + "Conflict Checking", | |
| 301 | + f"Repeated {total_checks} checks", | |
| 302 | + stats, | |
| 303 | + notes=cache_note, | |
| 304 | + ) | |
| 305 | + | |
| 306 | + def benchmark_list_operations_intensive(self): | |
| 307 | + """Test listing with large datasets""" | |
| 308 | + print("\n=== INTENSIVE List Operations ===") | |
| 309 | + | |
| 310 | + # Add more items if needed | |
| 311 | + print("Ensuring large dataset...") | |
| 312 | + groups = ["large_alpha", "large_beta", "large_gamma"] | |
| 313 | + for group in groups: | |
| 314 | + for i in range(100): | |
| 315 | + self.manager.add_alias(f"{group}_alias_{i}", f"echo {i}", group) | |
| 316 | + self.manager.add_env(f"{group}_env_{i}", f"{i}", group) | |
| 317 | + if i % 2 == 0: | |
| 318 | + self.manager.add_function( | |
| 319 | + f"{group}_func_{i}", f"echo func {i}", group | |
| 320 | + ) | |
| 321 | + | |
| 322 | + # Activate some groups | |
| 323 | + self.manager.activate_group("large_alpha") | |
| 324 | + self.manager.activate_group("large_beta") | |
| 325 | + | |
| 326 | + # Test listing all items | |
| 327 | + def list_all(): | |
| 328 | + items = self.manager.list_items() | |
| 329 | + return len(items) | |
| 330 | + | |
| 331 | + print("\nTesting list all items...") | |
| 332 | + stats = time_operation(list_all, iterations=20) | |
| 333 | + item_count = list_all() | |
| 334 | + print(f"List {item_count} items: {stats['mean']:.2f}ms") | |
| 335 | + print(f"Processing rate: {item_count/stats['mean']*1000:.0f} items/second") | |
| 336 | + self.record_result("List Operations", f"List all ({item_count} items)", stats) | |
| 337 | + | |
| 338 | + # Test listing by group | |
| 339 | + def list_large_group(): | |
| 340 | + items = self.manager.list_items("large_alpha") | |
| 341 | + return len(items) | |
| 342 | + | |
| 343 | + print("\nTesting list single large group...") | |
| 344 | + stats = time_operation(list_large_group, iterations=30) | |
| 345 | + group_count = list_large_group() | |
| 346 | + print(f"List group ({group_count} items): {stats['mean']:.2f}ms") | |
| 347 | + self.record_result( | |
| 348 | + "List Operations", f"List group ({group_count} items)", stats | |
| 349 | + ) | |
| 350 | + | |
| 351 | + def benchmark_batch_operations(self): | |
| 352 | + """Test batch operations if available""" | |
| 353 | + print("\n=== Batch Operations Performance ===") | |
| 354 | + | |
| 355 | + if not hasattr(self.manager, "add_items_batch"): | |
| 356 | + print("Batch operations not available in this version") | |
| 357 | + return | |
| 358 | + | |
| 359 | + # Test different batch sizes | |
| 360 | + batch_sizes = [10, 50, 100, 500] | |
| 361 | + | |
| 362 | + for size in batch_sizes: | |
| 363 | + print(f"\nTesting batch size: {size}") | |
| 364 | + | |
| 365 | + # Create batch | |
| 366 | + def add_batch(): | |
| 367 | + items = [ | |
| 368 | + { | |
| 369 | + "type": "alias", | |
| 370 | + "group": "batch_test", | |
| 371 | + "key": generate_random_key(f"batch{size}", 8), | |
| 372 | + "value": f"echo batch_{i}", | |
| 373 | + } | |
| 374 | + for i in range(size) | |
| 375 | + ] | |
| 376 | + self.manager.add_items_batch(items) | |
| 377 | + | |
| 378 | + stats = time_operation(add_batch, iterations=5) | |
| 379 | + print(f"Add batch of {size}: {stats['mean']:.2f}ms total") | |
| 380 | + print(f"Per-item in batch: {stats['mean']/size:.3f}ms") | |
| 381 | + print(f"Items/second: {size/stats['mean']*1000:.0f}") | |
| 382 | + | |
| 383 | + self.record_result("Batch Operations", f"Batch add ({size} items)", stats) | |
| 384 | + | |
| 385 | + # Compare with individual adds | |
| 386 | + def add_individual(): | |
| 387 | + for i in range(min(size, 20)): # Limit to 20 for time | |
| 388 | + key = generate_random_key(f"indiv{size}", 8) | |
| 389 | + self.manager.add_alias(key, f"echo {i}", "batch_test") | |
| 390 | + | |
| 391 | + stats_indiv = time_operation(add_individual, iterations=3) | |
| 392 | + per_item_indiv = stats_indiv["mean"] / min(size, 20) | |
| 393 | + | |
| 394 | + print(f"Individual adds: {per_item_indiv:.3f}ms per item") | |
| 395 | + print(f"Batch speedup: {per_item_indiv/(stats['mean']/size):.1f}x") | |
| 396 | + | |
| 397 | + def benchmark_file_generation_intensive(self): | |
| 398 | + """Test file generation with large groups""" | |
| 399 | + print("\n=== INTENSIVE File Generation ===") | |
| 400 | + | |
| 401 | + # Re-enable generation | |
| 402 | + if hasattr(self.manager, "_original_save_and_regenerate"): | |
| 403 | + self.manager._save_and_regenerate = ( | |
| 404 | + self.manager._original_save_and_regenerate | |
| 405 | + ) | |
| 406 | + | |
| 407 | + # Create groups of different sizes | |
| 408 | + group_sizes = {"small_gen": 10, "medium_gen": 100, "large_gen": 500} | |
| 409 | + | |
| 410 | + generator = Generator() | |
| 411 | + config = self.manager._get_config() | |
| 412 | + | |
| 413 | + if hasattr(generator, "set_config_for_shells"): | |
| 414 | + generator.set_config_for_shells(config) | |
| 415 | + | |
| 416 | + for group_name, size in group_sizes.items(): | |
| 417 | + print(f"\nTesting generation for {group_name} ({size} items)...") | |
| 418 | + | |
| 419 | + # Add items | |
| 420 | + for i in range(size): | |
| 421 | + self.manager.add_alias(f"alias_{i}", f"echo {i}", group_name) | |
| 422 | + if i % 3 == 0: | |
| 423 | + self.manager.add_env(f"var_{i}", f"value_{i}", group_name) | |
| 424 | + | |
| 425 | + # Test full generation | |
| 426 | + group = config.get_group(group_name) | |
| 427 | + if group: | |
| 428 | + | |
| 429 | + def generate_full(): | |
| 430 | + generator.generate_for_group(group) | |
| 431 | + | |
| 432 | + stats = time_operation(generate_full, iterations=5) | |
| 433 | + print(f"Full generation: {stats['mean']:.2f}ms") | |
| 434 | + self.record_result("File Generation", f"Generate {size} items", stats) | |
| 435 | + | |
| 436 | + # Test incremental if available | |
| 437 | + if hasattr(generator, "update_group_incrementally"): | |
| 438 | + # Single change | |
| 439 | + def incremental_single(): | |
| 440 | + changed_items = { | |
| 441 | + "added": [], | |
| 442 | + "modified": [("alias", "alias_0", "echo modified")], | |
| 443 | + "removed": [], | |
| 444 | + } | |
| 445 | + generator.update_group_incrementally(group, changed_items) | |
| 446 | + | |
| 447 | + stats_single = time_operation(incremental_single, iterations=10) | |
| 448 | + print(f"Incremental (1 change): {stats_single['mean']:.2f}ms") | |
| 449 | + print(f"Speedup: {stats['mean']/stats_single['mean']:.1f}x") | |
| 450 | + | |
| 451 | + # Multiple changes | |
| 452 | + def incremental_multi(): | |
| 453 | + changed_items = { | |
| 454 | + "added": [ | |
| 455 | + ("alias", f"new_{i}", f"echo new_{i}") for i in range(5) | |
| 456 | + ], | |
| 457 | + "modified": [ | |
| 458 | + ("alias", f"alias_{i}", f"echo mod_{i}") | |
| 459 | + for i in range(5) | |
| 460 | + ], | |
| 461 | + "removed": [ | |
| 462 | + ("alias", f"alias_{i+5}", f"echo {i+5}") | |
| 463 | + for i in range(5) | |
| 464 | + ], | |
| 465 | + } | |
| 466 | + generator.update_group_incrementally(group, changed_items) | |
| 467 | + | |
| 468 | + stats_multi = time_operation(incremental_multi, iterations=10) | |
| 469 | + print(f"Incremental (15 changes): {stats_multi['mean']:.2f}ms") | |
| 470 | + print(f"Speedup: {stats['mean']/stats_multi['mean']:.1f}x") | |
| 471 | + | |
| 472 | + self.record_result( | |
| 473 | + "File Generation", | |
| 474 | + f"Incremental {size} items (1 change)", | |
| 475 | + stats_single, | |
| 476 | + ) | |
| 477 | + self.record_result( | |
| 478 | + "File Generation", | |
| 479 | + f"Incremental {size} items (15 changes)", | |
| 480 | + stats_multi, | |
| 481 | + ) | |
| 482 | + | |
| 483 | + def benchmark_stress_test(self): | |
| 484 | + """Ultimate stress test - simulate real heavy usage""" | |
| 485 | + print("\n=== STRESS TEST - Simulating Heavy Usage ===") | |
| 486 | + | |
| 487 | + print("This simulates a power user with many groups and items...") | |
| 488 | + | |
| 489 | + # Setup: Create a realistic large configuration | |
| 490 | + departments = ["engineering", "devops", "data", "security", "qa"] | |
| 491 | + environments = ["dev", "staging", "prod"] | |
| 492 | + | |
| 493 | + total_start = time.perf_counter() | |
| 494 | + | |
| 495 | + # Create many items | |
| 496 | + print("Creating large configuration...") | |
| 497 | + for dept in departments: | |
| 498 | + for env in environments: | |
| 499 | + group_name = f"{dept}_{env}" | |
| 500 | + | |
| 501 | + # Add various items | |
| 502 | + for i in range(50): | |
| 503 | + self.manager.add_alias( | |
| 504 | + f"{dept}_cmd_{i}", f"{dept} command {i}", group_name | |
| 505 | + ) | |
| 506 | + | |
| 507 | + for i in range(30): | |
| 508 | + self.manager.add_env( | |
| 509 | + f"{dept.upper()}_VAR_{i}", f"{env}_{i}", group_name | |
| 510 | + ) | |
| 511 | + | |
| 512 | + for i in range(10): | |
| 513 | + func_body = f"echo Running {dept} function {i} in {env}" | |
| 514 | + self.manager.add_function(f"{dept}_func_{i}", func_body, group_name) | |
| 515 | + | |
| 516 | + creation_time = (time.perf_counter() - total_start) * 1000 | |
| 517 | + | |
| 518 | + # Now run typical operations | |
| 519 | + print("\nRunning typical operations...") | |
| 520 | + | |
| 521 | + # 1. List everything | |
| 522 | + list_start = time.perf_counter() | |
| 523 | + all_items = self.manager.list_items() | |
| 524 | + list_time = (time.perf_counter() - list_start) * 1000 | |
| 525 | + | |
| 526 | + # 2. Check many conflicts | |
| 527 | + conflict_start = time.perf_counter() | |
| 528 | + for _ in range(100): | |
| 529 | + key = generate_random_key("check", 10) | |
| 530 | + self.manager.check_conflicts("alias", key, "engineering_dev") | |
| 531 | + conflict_time = (time.perf_counter() - conflict_start) * 1000 | |
| 532 | + | |
| 533 | + # 3. Activate/deactivate groups | |
| 534 | + activate_start = time.perf_counter() | |
| 535 | + for dept in departments: | |
| 536 | + self.manager.activate_group(f"{dept}_dev") | |
| 537 | + for dept in departments[:3]: | |
| 538 | + self.manager.deactivate_group(f"{dept}_dev") | |
| 539 | + activate_time = (time.perf_counter() - activate_start) * 1000 | |
| 540 | + | |
| 541 | + # 4. Get status | |
| 542 | + status_start = time.perf_counter() | |
| 543 | + status = self.manager.get_status() | |
| 544 | + status_time = (time.perf_counter() - status_start) * 1000 | |
| 545 | + | |
| 546 | + # Results | |
| 547 | + print(f"\nStress Test Results:") | |
| 548 | + print(f"Total items created: {len(all_items)}") | |
| 549 | + print( | |
| 550 | + f"Creation time: {creation_time:.0f}ms ({len(all_items)/creation_time*1000:.0f} items/sec)" | |
| 551 | + ) | |
| 552 | + print(f"List all items: {list_time:.1f}ms") | |
| 553 | + print( | |
| 554 | + f"100 conflict checks: {conflict_time:.1f}ms ({conflict_time/100:.2f}ms each)" | |
| 555 | + ) | |
| 556 | + print(f"Group operations: {activate_time:.1f}ms") | |
| 557 | + print(f"Status check: {status_time:.1f}ms") | |
| 558 | + | |
| 559 | + self.record_result( | |
| 560 | + "Stress Test", | |
| 561 | + f"Create {len(all_items)} items", | |
| 562 | + { | |
| 563 | + "mean": creation_time, | |
| 564 | + "median": creation_time, | |
| 565 | + "min": creation_time, | |
| 566 | + "max": creation_time, | |
| 567 | + "stdev": 0, | |
| 568 | + }, | |
| 569 | + ) | |
| 570 | + self.record_result( | |
| 571 | + "Stress Test", | |
| 572 | + f"List {len(all_items)} items", | |
| 573 | + { | |
| 574 | + "mean": list_time, | |
| 575 | + "median": list_time, | |
| 576 | + "min": list_time, | |
| 577 | + "max": list_time, | |
| 578 | + "stdev": 0, | |
| 579 | + }, | |
| 580 | + ) | |
| 581 | + self.record_result( | |
| 582 | + "Stress Test", | |
| 583 | + "100 conflict checks", | |
| 584 | + { | |
| 585 | + "mean": conflict_time, | |
| 586 | + "median": conflict_time, | |
| 587 | + "min": conflict_time, | |
| 588 | + "max": conflict_time, | |
| 589 | + "stdev": 0, | |
| 590 | + }, | |
| 591 | + ) | |
| 592 | + | |
| 593 | + def write_results_to_file(self): | |
| 594 | + """Write comprehensive results""" | |
| 595 | + timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") | |
| 596 | + version_suffix = "opt" if OPTIMIZED_VERSION else "std" | |
| 597 | + filename = f"bench-intensive-{version_suffix}-{timestamp}.out" | |
| 598 | + | |
| 599 | + with open(filename, "w") as f: | |
| 600 | + f.write("SHTICK INTENSIVE PERFORMANCE BENCHMARK RESULTS\n") | |
| 601 | + f.write(f"Version: {self.version_info['version'].upper()}\n") | |
| 602 | + f.write(f"Timestamp: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") | |
| 603 | + f.write("=" * 80 + "\n\n") | |
| 604 | + | |
| 605 | + # Summary table | |
| 606 | + f.write("PERFORMANCE SUMMARY (all times in milliseconds)\n") | |
| 607 | + f.write("-" * 80 + "\n") | |
| 608 | + f.write( | |
| 609 | + f"{'Category':<25} {'Operation':<40} {'Mean':>10} {'Items/sec':>12}\n" | |
| 610 | + ) | |
| 611 | + f.write("-" * 80 + "\n") | |
| 612 | + | |
| 613 | + for category, operations in sorted(self.results.items()): | |
| 614 | + for operation, stats in sorted(operations.items()): | |
| 615 | + # Calculate items/sec if applicable | |
| 616 | + items_per_sec = "" | |
| 617 | + if "items)" in operation: | |
| 618 | + # Extract number from operation name | |
| 619 | + import re | |
| 620 | + | |
| 621 | + match = re.search(r"(\d+) items", operation) | |
| 622 | + if match: | |
| 623 | + item_count = int(match.group(1)) | |
| 624 | + items_per_sec = f"{item_count/stats['mean_ms']*1000:,.0f}" | |
| 625 | + | |
| 626 | + f.write( | |
| 627 | + f"{category:<25} {operation:<40} {stats['mean_ms']:>10.2f} {items_per_sec:>12}\n" | |
| 628 | + ) | |
| 629 | + | |
| 630 | + # Version comparison section | |
| 631 | + f.write("\n\nOPTIMIZATION IMPACT\n") | |
| 632 | + f.write("=" * 80 + "\n") | |
| 633 | + | |
| 634 | + if OPTIMIZED_VERSION: | |
| 635 | + f.write("This version includes optimizations:\n") | |
| 636 | + f.write("✓ Pre-compiled regex patterns for key validation\n") | |
| 637 | + f.write("✓ LRU caching for conflict checks and lookups\n") | |
| 638 | + f.write("✓ Incremental file generation\n") | |
| 639 | + f.write("✓ Cached settings and shell detection\n") | |
| 640 | + else: | |
| 641 | + f.write("This is the standard version without optimizations.\n") | |
| 642 | + f.write("Compare with optimized version to see improvements.\n") | |
| 643 | + | |
| 644 | + # Detailed results | |
| 645 | + f.write("\n\nDETAILED RESULTS\n") | |
| 646 | + f.write("=" * 80 + "\n") | |
| 647 | + | |
| 648 | + for category, operations in sorted(self.results.items()): | |
| 649 | + f.write(f"\n{category}:\n") | |
| 650 | + f.write("-" * len(category) + "\n") | |
| 651 | + for operation, stats in sorted(operations.items()): | |
| 652 | + f.write(f" {operation}:\n") | |
| 653 | + f.write(f" Mean: {stats['mean_ms']:.3f}ms\n") | |
| 654 | + f.write(f" Median: {stats['median_ms']:.3f}ms\n") | |
| 655 | + f.write(f" Min: {stats['min_ms']:.3f}ms\n") | |
| 656 | + f.write(f" Max: {stats['max_ms']:.3f}ms\n") | |
| 657 | + if stats["stdev_ms"] > 0: | |
| 658 | + f.write(f" StdDev: {stats['stdev_ms']:.3f}ms\n") | |
| 659 | + if stats.get("notes"): | |
| 660 | + f.write(f" Notes: {stats['notes']}\n") | |
| 661 | + f.write("\n") | |
| 662 | + | |
| 663 | + # JSON for analysis | |
| 664 | + json_data = { | |
| 665 | + "timestamp": datetime.now().isoformat(), | |
| 666 | + "version": self.version_info, | |
| 667 | + "results": self.results, | |
| 668 | + } | |
| 669 | + | |
| 670 | + f.write("\n\nJSON DATA\n") | |
| 671 | + f.write("=" * 80 + "\n") | |
| 672 | + f.write(json.dumps(json_data, indent=2)) | |
| 673 | + | |
| 674 | + print(f"\n✓ Results written to: {filename}") | |
| 675 | + return filename | |
| 676 | + | |
| 677 | + | |
| 678 | +def main(): | |
| 679 | + """Run intensive benchmarks""" | |
| 680 | + print("=" * 70) | |
| 681 | + print("SHTICK INTENSIVE PERFORMANCE BENCHMARKS") | |
| 682 | + print(f"Version: {'OPTIMIZED' if OPTIMIZED_VERSION else 'STANDARD'}") | |
| 683 | + print("=" * 70) | |
| 684 | + print("Running with larger datasets to show real performance differences...") | |
| 685 | + print() | |
| 686 | + | |
| 687 | + benchmark = IntensiveBenchmark() | |
| 688 | + | |
| 689 | + try: | |
| 690 | + benchmark.setup() | |
| 691 | + | |
| 692 | + # Run intensive benchmarks | |
| 693 | + benchmark.benchmark_key_validation_intensive() | |
| 694 | + benchmark.benchmark_conflict_checking_intensive() | |
| 695 | + benchmark.benchmark_list_operations_intensive() | |
| 696 | + benchmark.benchmark_batch_operations() | |
| 697 | + benchmark.benchmark_file_generation_intensive() | |
| 698 | + benchmark.benchmark_stress_test() | |
| 699 | + | |
| 700 | + # Summary | |
| 701 | + print("\n" + "=" * 70) | |
| 702 | + print("INTENSIVE BENCHMARK COMPLETE") | |
| 703 | + print("=" * 70) | |
| 704 | + | |
| 705 | + # Write results | |
| 706 | + output_file = benchmark.write_results_to_file() | |
| 707 | + | |
| 708 | + print("\nNext steps:") | |
| 709 | + print("1. Run this on the other branch to compare") | |
| 710 | + print("2. Compare results:") | |
| 711 | + print(f" diff bench-intensive-std-*.out bench-intensive-opt-*.out") | |
| 712 | + print("3. Look for:") | |
| 713 | + print(" - Conflict checking improvements (caching)") | |
| 714 | + print(" - Batch operation speedups") | |
| 715 | + print(" - File generation improvements (incremental)") | |
| 716 | + print(" - Overall items/second processing rates") | |
| 717 | + | |
| 718 | + finally: | |
| 719 | + benchmark.cleanup() | |
| 720 | + print("\n✓ Cleanup complete") | |
| 721 | + | |
| 722 | + | |
| 723 | +if __name__ == "__main__": | |
| 724 | + main() | |
src/shtick/diag-optimized.txtadded@@ -0,0 +1,107 @@ | ||
| 1 | +✓ Successfully imported shtick modules | |
| 2 | +✓ OPTIMIZED version detected | |
| 3 | +============================================================ | |
| 4 | +PERFORMANCE DIAGNOSTICS - OPTIMIZED VERSION | |
| 5 | +============================================================ | |
| 6 | + | |
| 7 | +Profiling: Single alias add | |
| 8 | +-------------------------------------------------- | |
| 9 | +Time: 5.63ms | |
| 10 | + | |
| 11 | +Top 10 time consumers: | |
| 12 | + 1695 function calls (1687 primitive calls) in 0.002 seconds | |
| 13 | + | |
| 14 | + Ordered by: cumulative time | |
| 15 | + List reduced from 154 to 10 due to restriction <10> | |
| 16 | + | |
| 17 | + ncalls tottime percall cumtime percall filename:lineno(function) | |
| 18 | + 1 0.000 0.000 0.002 0.002 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/performanc_diagnostic.py:88(add_single) | |
| 19 | + 1 0.000 0.000 0.002 0.002 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:218(add_alias) | |
| 20 | + 1 0.000 0.000 0.002 0.002 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:343(_add_item) | |
| 21 | + 1 0.000 0.000 0.002 0.002 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:107(_save_and_regenerate) | |
| 22 | + 1 0.000 0.000 0.002 0.002 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/generator.py:381(generate_loader) | |
| 23 | + 19 0.000 0.000 0.001 0.000 {built-in method io.open} | |
| 24 | + 9 0.000 0.000 0.001 0.000 /Users/matthewwolffe/.asdf/installs/python/3.11.6/lib/python3.11/tempfile.py:522(NamedTemporaryFile) | |
| 25 | + 1 0.000 0.000 0.001 0.001 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/generator.py:96(generate_for_group) | |
| 26 | + 9 0.001 0.000 0.001 0.000 {built-in method posix.replace} | |
| 27 | + 9 0.000 0.000 0.001 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/generator.py:179(_generate_group_file) | |
| 28 | + | |
| 29 | + | |
| 30 | + | |
| 31 | +Profiling: Batch add (100 items) | |
| 32 | +-------------------------------------------------- | |
| 33 | +Time: 3.29ms | |
| 34 | + | |
| 35 | +Top 10 time consumers: | |
| 36 | + 15000 function calls (14788 primitive calls) in 0.006 seconds | |
| 37 | + | |
| 38 | + Ordered by: cumulative time | |
| 39 | + List reduced from 154 to 10 due to restriction <10> | |
| 40 | + | |
| 41 | + ncalls tottime percall cumtime percall filename:lineno(function) | |
| 42 | + 1 0.000 0.000 0.006 0.006 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/performanc_diagnostic.py:97(add_batch) | |
| 43 | + 1 0.000 0.000 0.006 0.006 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:441(add_items_batch) | |
| 44 | + 1 0.000 0.000 0.004 0.004 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:107(_save_and_regenerate) | |
| 45 | + 2 0.000 0.000 0.002 0.001 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/generator.py:96(generate_for_group) | |
| 46 | + 18 0.000 0.000 0.002 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/generator.py:179(_generate_group_file) | |
| 47 | + 1 0.000 0.000 0.001 0.001 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/generator.py:381(generate_loader) | |
| 48 | + 100 0.000 0.000 0.001 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/config.py:226(is_group_active) | |
| 49 | + 101 0.000 0.000 0.001 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/config.py:159(load_active_groups) | |
| 50 | + 28 0.000 0.000 0.001 0.000 {built-in method io.open} | |
| 51 | + 9 0.001 0.000 0.001 0.000 {built-in method posix.replace} | |
| 52 | + | |
| 53 | + | |
| 54 | + | |
| 55 | +Profiling: Conflict checking (50 checks) | |
| 56 | +-------------------------------------------------- | |
| 57 | +Time: 0.03ms | |
| 58 | + | |
| 59 | +Top 10 time consumers: | |
| 60 | + 102 function calls in 0.000 seconds | |
| 61 | + | |
| 62 | + Ordered by: cumulative time | |
| 63 | + | |
| 64 | + ncalls tottime percall cumtime percall filename:lineno(function) | |
| 65 | + 1 0.000 0.000 0.000 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/performanc_diagnostic.py:111(check_conflicts) | |
| 66 | + 50 0.000 0.000 0.000 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:168(check_conflicts) | |
| 67 | + 50 0.000 0.000 0.000 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:184(<listcomp>) | |
| 68 | + 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} | |
| 69 | + | |
| 70 | + | |
| 71 | + | |
| 72 | +Profiling: List all items | |
| 73 | +-------------------------------------------------- | |
| 74 | +Time: 0.82ms | |
| 75 | + | |
| 76 | +Top 10 time consumers: | |
| 77 | + 6708 function calls in 0.002 seconds | |
| 78 | + | |
| 79 | + Ordered by: cumulative time | |
| 80 | + List reduced from 33 to 10 due to restriction <10> | |
| 81 | + | |
| 82 | + ncalls tottime percall cumtime percall filename:lineno(function) | |
| 83 | + 1 0.000 0.000 0.002 0.002 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/performanc_diagnostic.py:118(list_all) | |
| 84 | + 1 0.000 0.000 0.002 0.002 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:704(list_items) | |
| 85 | + 202 0.000 0.000 0.002 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/config.py:226(is_group_active) | |
| 86 | + 202 0.000 0.000 0.002 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/config.py:159(load_active_groups) | |
| 87 | + 202 0.000 0.000 0.001 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/config.py:115(get_active_groups_file) | |
| 88 | + 202 0.000 0.000 0.001 0.000 <frozen posixpath>:229(expanduser) | |
| 89 | + 404 0.001 0.000 0.001 0.000 {built-in method posix.stat} | |
| 90 | + 404 0.000 0.000 0.000 0.000 <frozen os>:674(__getitem__) | |
| 91 | + 202 0.000 0.000 0.000 0.000 <frozen genericpath>:16(exists) | |
| 92 | + 202 0.000 0.000 0.000 0.000 <frozen genericpath>:53(getmtime) | |
| 93 | + | |
| 94 | + | |
| 95 | + | |
| 96 | +============================================================ | |
| 97 | +OPTIMIZED VERSION SPECIFIC CHECKS | |
| 98 | +============================================================ | |
| 99 | + | |
| 100 | +LRU Cache stats: | |
| 101 | + Hits: 49 | |
| 102 | + Misses: 1 | |
| 103 | + Hit rate: 98.0% | |
| 104 | + | |
| 105 | +Cache clears during 10 adds: 10 | |
| 106 | + | |
| 107 | +✓ Diagnostic complete | |
src/shtick/diag-standard.txtadded@@ -0,0 +1,100 @@ | ||
| 1 | +✓ Successfully imported shtick modules | |
| 2 | +✓ STANDARD version detected | |
| 3 | +============================================================ | |
| 4 | +PERFORMANCE DIAGNOSTICS - STANDARD VERSION | |
| 5 | +============================================================ | |
| 6 | + | |
| 7 | +Profiling: Single alias add | |
| 8 | +-------------------------------------------------- | |
| 9 | +Time: 3.25ms | |
| 10 | + | |
| 11 | +Top 10 time consumers: | |
| 12 | + 699 function calls in 0.001 seconds | |
| 13 | + | |
| 14 | + Ordered by: cumulative time | |
| 15 | + List reduced from 90 to 10 due to restriction <10> | |
| 16 | + | |
| 17 | + ncalls tottime percall cumtime percall filename:lineno(function) | |
| 18 | + 1 0.000 0.000 0.001 0.001 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/performanc_diagnostic.py:88(add_single) | |
| 19 | + 1 0.000 0.000 0.001 0.001 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:133(add_alias) | |
| 20 | + 1 0.000 0.000 0.001 0.001 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:258(_add_item) | |
| 21 | + 1 0.000 0.000 0.001 0.001 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:78(_save_and_regenerate) | |
| 22 | + 1 0.000 0.000 0.001 0.001 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/generator.py:52(generate_for_group) | |
| 23 | + 9 0.000 0.000 0.000 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/generator.py:77(_generate_group_file) | |
| 24 | + 1 0.000 0.000 0.000 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/generator.py:152(generate_loader) | |
| 25 | + 19 0.000 0.000 0.000 0.000 {built-in method io.open} | |
| 26 | + 19 0.000 0.000 0.000 0.000 {method '__exit__' of '_io._IOBase' objects} | |
| 27 | + 1 0.000 0.000 0.000 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/config.py:258(save) | |
| 28 | + | |
| 29 | + | |
| 30 | + | |
| 31 | +Profiling: Batch add (100 items) | |
| 32 | +-------------------------------------------------- | |
| 33 | +Time: 2.10ms | |
| 34 | + | |
| 35 | +Top 10 time consumers: | |
| 36 | + 8972 function calls in 0.004 seconds | |
| 37 | + | |
| 38 | + Ordered by: cumulative time | |
| 39 | + List reduced from 93 to 10 due to restriction <10> | |
| 40 | + | |
| 41 | + ncalls tottime percall cumtime percall filename:lineno(function) | |
| 42 | + 1 0.000 0.000 0.004 0.004 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/performanc_diagnostic.py:97(add_batch) | |
| 43 | + 1 0.000 0.000 0.004 0.004 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:321(add_items_batch) | |
| 44 | + 1 0.000 0.000 0.002 0.002 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:78(_save_and_regenerate) | |
| 45 | + 2 0.000 0.000 0.001 0.001 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/generator.py:52(generate_for_group) | |
| 46 | + 18 0.000 0.000 0.001 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/generator.py:77(_generate_group_file) | |
| 47 | + 100 0.000 0.000 0.001 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/config.py:177(is_group_active) | |
| 48 | + 101 0.000 0.000 0.001 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/config.py:110(load_active_groups) | |
| 49 | + 28 0.001 0.000 0.001 0.000 {built-in method io.open} | |
| 50 | + 28 0.001 0.000 0.001 0.000 {method '__exit__' of '_io._IOBase' objects} | |
| 51 | + 101 0.000 0.000 0.000 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/config.py:84(get_active_groups_file) | |
| 52 | + | |
| 53 | + | |
| 54 | + | |
| 55 | +Profiling: Conflict checking (50 checks) | |
| 56 | +-------------------------------------------------- | |
| 57 | +Time: 0.04ms | |
| 58 | + | |
| 59 | +Top 10 time consumers: | |
| 60 | + 702 function calls in 0.000 seconds | |
| 61 | + | |
| 62 | + Ordered by: cumulative time | |
| 63 | + | |
| 64 | + ncalls tottime percall cumtime percall filename:lineno(function) | |
| 65 | + 1 0.000 0.000 0.000 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/performanc_diagnostic.py:111(check_conflicts) | |
| 66 | + 50 0.000 0.000 0.000 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:97(check_conflicts) | |
| 67 | + 150 0.000 0.000 0.000 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/config.py:45(has_item) | |
| 68 | + 150 0.000 0.000 0.000 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/config.py:24(get_items) | |
| 69 | + 150 0.000 0.000 0.000 0.000 {built-in method builtins.getattr} | |
| 70 | + 150 0.000 0.000 0.000 0.000 {method 'get' of 'dict' objects} | |
| 71 | + 50 0.000 0.000 0.000 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:72(_get_config) | |
| 72 | + 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} | |
| 73 | + | |
| 74 | + | |
| 75 | + | |
| 76 | +Profiling: List all items | |
| 77 | +-------------------------------------------------- | |
| 78 | +Time: 0.84ms | |
| 79 | + | |
| 80 | +Top 10 time consumers: | |
| 81 | + 6708 function calls in 0.002 seconds | |
| 82 | + | |
| 83 | + Ordered by: cumulative time | |
| 84 | + List reduced from 33 to 10 due to restriction <10> | |
| 85 | + | |
| 86 | + ncalls tottime percall cumtime percall filename:lineno(function) | |
| 87 | + 1 0.000 0.000 0.002 0.002 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/performanc_diagnostic.py:118(list_all) | |
| 88 | + 1 0.000 0.000 0.002 0.002 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/shtick.py:533(list_items) | |
| 89 | + 202 0.000 0.000 0.002 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/config.py:177(is_group_active) | |
| 90 | + 202 0.000 0.000 0.002 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/config.py:110(load_active_groups) | |
| 91 | + 202 0.000 0.000 0.001 0.000 /Users/matthewwolffe/Documents/GithubOrgs/tenselyflow/shtickpy/src/shtick/config.py:84(get_active_groups_file) | |
| 92 | + 202 0.000 0.000 0.001 0.000 <frozen posixpath>:229(expanduser) | |
| 93 | + 404 0.000 0.000 0.000 0.000 {built-in method posix.stat} | |
| 94 | + 404 0.000 0.000 0.000 0.000 <frozen os>:674(__getitem__) | |
| 95 | + 202 0.000 0.000 0.000 0.000 <frozen genericpath>:16(exists) | |
| 96 | + 202 0.000 0.000 0.000 0.000 <frozen genericpath>:53(getmtime) | |
| 97 | + | |
| 98 | + | |
| 99 | + | |
| 100 | +✓ Diagnostic complete | |
src/shtick/performanc_diagnostic.pyadded@@ -0,0 +1,184 @@ | ||
| 1 | +#!/usr/bin/env python3 | |
| 2 | +""" | |
| 3 | +Diagnostic script to identify performance bottlenecks | |
| 4 | +""" | |
| 5 | + | |
| 6 | +import time | |
| 7 | +import tempfile | |
| 8 | +import os | |
| 9 | +import sys | |
| 10 | +import cProfile | |
| 11 | +import pstats | |
| 12 | +from pathlib import Path | |
| 13 | + | |
| 14 | +# Fix imports | |
| 15 | +current_file = Path(__file__).resolve() | |
| 16 | +shtick_dir = current_file.parent | |
| 17 | +src_dir = shtick_dir.parent | |
| 18 | +project_root = src_dir.parent | |
| 19 | + | |
| 20 | +sys.path.insert(0, str(shtick_dir)) | |
| 21 | +sys.path.insert(0, str(src_dir)) | |
| 22 | +sys.path.insert(0, str(project_root)) | |
| 23 | + | |
| 24 | +try: | |
| 25 | + from shtick import ShtickManager | |
| 26 | + from shtick.config import Config | |
| 27 | + | |
| 28 | + print("✓ Successfully imported shtick modules") | |
| 29 | + | |
| 30 | + # Check version | |
| 31 | + if hasattr(ShtickManager, "_get_all_items_by_type"): | |
| 32 | + print("✓ OPTIMIZED version detected") | |
| 33 | + VERSION = "optimized" | |
| 34 | + else: | |
| 35 | + print("✓ STANDARD version detected") | |
| 36 | + VERSION = "standard" | |
| 37 | +except ImportError as e: | |
| 38 | + print(f"✗ Import error: {e}") | |
| 39 | + sys.exit(1) | |
| 40 | + | |
| 41 | + | |
| 42 | +def profile_operation(func, name): | |
| 43 | + """Profile a single operation""" | |
| 44 | + print(f"\nProfiling: {name}") | |
| 45 | + print("-" * 50) | |
| 46 | + | |
| 47 | + # Time it | |
| 48 | + start = time.perf_counter() | |
| 49 | + func() | |
| 50 | + elapsed = (time.perf_counter() - start) * 1000 | |
| 51 | + print(f"Time: {elapsed:.2f}ms") | |
| 52 | + | |
| 53 | + # Profile it | |
| 54 | + profiler = cProfile.Profile() | |
| 55 | + profiler.enable() | |
| 56 | + func() | |
| 57 | + profiler.disable() | |
| 58 | + | |
| 59 | + # Show top time consumers | |
| 60 | + stats = pstats.Stats(profiler) | |
| 61 | + stats.sort_stats("cumulative") | |
| 62 | + print("\nTop 10 time consumers:") | |
| 63 | + stats.print_stats(10) | |
| 64 | + | |
| 65 | + return elapsed | |
| 66 | + | |
| 67 | + | |
| 68 | +def main(): | |
| 69 | + """Run diagnostic tests""" | |
| 70 | + print("=" * 60) | |
| 71 | + print(f"PERFORMANCE DIAGNOSTICS - {VERSION.upper()} VERSION") | |
| 72 | + print("=" * 60) | |
| 73 | + | |
| 74 | + # Setup | |
| 75 | + temp_dir = tempfile.mkdtemp(prefix="shtick_diag_") | |
| 76 | + config_path = os.path.join(temp_dir, "config.toml") | |
| 77 | + | |
| 78 | + with open(config_path, "w") as f: | |
| 79 | + f.write("[persistent]\n") | |
| 80 | + | |
| 81 | + # Disable logging | |
| 82 | + import logging | |
| 83 | + | |
| 84 | + logging.disable(logging.CRITICAL) | |
| 85 | + | |
| 86 | + manager = ShtickManager(config_path=config_path, debug=False) | |
| 87 | + | |
| 88 | + # Test 1: Single add operation | |
| 89 | + counter = 0 | |
| 90 | + | |
| 91 | + def add_single(): | |
| 92 | + nonlocal counter | |
| 93 | + manager.add_alias(f"test_{counter}", f"echo {counter}", "test") | |
| 94 | + counter += 1 | |
| 95 | + | |
| 96 | + profile_operation(add_single, "Single alias add") | |
| 97 | + | |
| 98 | + # Test 2: Batch operation (if available) | |
| 99 | + if hasattr(manager, "add_items_batch"): | |
| 100 | + | |
| 101 | + def add_batch(): | |
| 102 | + items = [ | |
| 103 | + { | |
| 104 | + "type": "alias", | |
| 105 | + "group": "batch", | |
| 106 | + "key": f"batch_{i}", | |
| 107 | + "value": f"echo {i}", | |
| 108 | + } | |
| 109 | + for i in range(100) | |
| 110 | + ] | |
| 111 | + manager.add_items_batch(items) | |
| 112 | + | |
| 113 | + profile_operation(add_batch, "Batch add (100 items)") | |
| 114 | + | |
| 115 | + # Test 3: Conflict checking | |
| 116 | + # First add some items | |
| 117 | + for i in range(100): | |
| 118 | + manager.add_alias(f"existing_{i}", f"echo {i}", "conflicts") | |
| 119 | + | |
| 120 | + def check_conflicts(): | |
| 121 | + for i in range(50): | |
| 122 | + manager.check_conflicts("alias", f"new_{i}", "conflicts") | |
| 123 | + | |
| 124 | + profile_operation(check_conflicts, "Conflict checking (50 checks)") | |
| 125 | + | |
| 126 | + # Test 4: List operations | |
| 127 | + def list_all(): | |
| 128 | + items = manager.list_items() | |
| 129 | + return len(items) | |
| 130 | + | |
| 131 | + profile_operation(list_all, "List all items") | |
| 132 | + | |
| 133 | + # Test 5: Specific to optimized version | |
| 134 | + if VERSION == "optimized": | |
| 135 | + print("\n" + "=" * 60) | |
| 136 | + print("OPTIMIZED VERSION SPECIFIC CHECKS") | |
| 137 | + print("=" * 60) | |
| 138 | + | |
| 139 | + # Check if caches are working | |
| 140 | + if hasattr(manager, "_get_all_items_by_type") and hasattr( | |
| 141 | + manager._get_all_items_by_type, "cache_info" | |
| 142 | + ): | |
| 143 | + cache_info = manager._get_all_items_by_type.cache_info() | |
| 144 | + print(f"\nLRU Cache stats:") | |
| 145 | + print(f" Hits: {cache_info.hits}") | |
| 146 | + print(f" Misses: {cache_info.misses}") | |
| 147 | + print( | |
| 148 | + f" Hit rate: {cache_info.hits/(cache_info.hits+cache_info.misses)*100:.1f}%" | |
| 149 | + if cache_info.hits + cache_info.misses > 0 | |
| 150 | + else " Hit rate: N/A" | |
| 151 | + ) | |
| 152 | + | |
| 153 | + # Check if we're clearing caches too often | |
| 154 | + clear_count = 0 | |
| 155 | + original_clear = None | |
| 156 | + if hasattr(manager, "_clear_caches"): | |
| 157 | + original_clear = manager._clear_caches | |
| 158 | + | |
| 159 | + def counting_clear(): | |
| 160 | + nonlocal clear_count | |
| 161 | + clear_count += 1 | |
| 162 | + original_clear() | |
| 163 | + | |
| 164 | + manager._clear_caches = counting_clear | |
| 165 | + | |
| 166 | + # Do some operations | |
| 167 | + for i in range(10): | |
| 168 | + manager.add_alias(f"cache_test_{i}", f"echo {i}", "cache_test") | |
| 169 | + | |
| 170 | + print(f"\nCache clears during 10 adds: {clear_count}") | |
| 171 | + | |
| 172 | + if clear_count > 10: | |
| 173 | + print("⚠️ WARNING: Caches are being cleared too frequently!") | |
| 174 | + | |
| 175 | + # Cleanup | |
| 176 | + import shutil | |
| 177 | + | |
| 178 | + shutil.rmtree(temp_dir) | |
| 179 | + | |
| 180 | + print("\n✓ Diagnostic complete") | |
| 181 | + | |
| 182 | + | |
| 183 | +if __name__ == "__main__": | |
| 184 | + main() | |