diff --git a/neurips23/runbooks/gen_expiration_time_runbook.py b/neurips23/runbooks/gen_expiration_time_runbook.py index 400b0973..008eb038 100644 --- a/neurips23/runbooks/gen_expiration_time_runbook.py +++ b/neurips23/runbooks/gen_expiration_time_runbook.py @@ -224,7 +224,29 @@ def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ra dataset_name = 'msturingpq-100K' dataset_size = 100000 max_t = 100 -gt_url = None -max_points = 28000 +gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturing-100K_expiration_time_runbook.yaml" +max_points = None gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, False, gt_url, True, max_points) +ratios = (0, 4, 18) +timesteps = (0, 50, 5) +seed = 5554 +dataset_file = 'msturingpq-100K_expiration_time_replace_delete_runbook.yaml' +dataset_name = 'msturingpq-100K' +dataset_size = 100000 +max_t = 100 +gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturing-100K_expiration_time_replace_delete_runbook.yaml" +max_points = None +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, gt_url, True, max_points) + +ratios = (0, 4, 18) +timesteps = (0, 50, 5) +seed = 5554 +dataset_file = 'msturingpq-100K_expiration_time_replace_only_runbook.yaml' +dataset_name = 'msturingpq-100K' +dataset_size = 100000 +max_t = 100 +gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturing-100K_expiration_time_replace_only_runbook.yaml" +max_points = None +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, gt_url, False, max_points) + diff --git a/neurips23/runbooks/msturingpq-100K_expiration_time_replace_delete_runbook.yaml b/neurips23/runbooks/msturingpq-100K_expiration_time_replace_delete_runbook.yaml new file mode 100644 index 00000000..2e123602 --- /dev/null +++ b/neurips23/runbooks/msturingpq-100K_expiration_time_replace_delete_runbook.yaml @@ -0,0 +1,1489 @@ +msturingpq-100K: + 1: + end: 834 + operation: insert + start: 0 + 2: + operation: search + 3: + end: 1777 + operation: insert + start: 1000 + 4: + operation: search + 5: + end: 2772 + operation: insert + start: 2000 + 6: + operation: search + 7: + end: 3664 + operation: insert + start: 3000 + 8: + operation: search + 9: + end: 4735 + operation: insert + start: 4000 + 10: + ids_end: 3000 + ids_start: 2772 + operation: replace + tags_end: 2228 + tags_start: 2000 + 11: + operation: search + 12: + end: 5755 + operation: insert + start: 5000 + 13: + ids_end: 4000 + ids_start: 3664 + operation: replace + tags_end: 3336 + tags_start: 3000 + 14: + operation: search + 15: + end: 6604 + operation: insert + start: 6000 + 16: + ids_end: 5000 + ids_start: 4735 + operation: replace + tags_end: 4265 + tags_start: 4000 + 17: + operation: search + 18: + end: 7644 + operation: insert + start: 7000 + 19: + end: 2772 + operation: delete + start: 2000 + 20: + ids_end: 6000 + ids_start: 5755 + operation: replace + tags_end: 5245 + tags_start: 5000 + 21: + operation: search + 22: + end: 8773 + operation: insert + start: 8000 + 23: + end: 3664 + operation: delete + start: 3000 + 24: + ids_end: 7000 + ids_start: 6604 + operation: replace + tags_end: 6396 + tags_start: 6000 + 25: + operation: search + 26: + end: 9821 + operation: insert + start: 9000 + 27: + end: 4735 + operation: delete + start: 4000 + 28: + ids_end: 8000 + ids_start: 7644 + operation: replace + tags_end: 7356 + tags_start: 7000 + 29: + operation: search + 30: + end: 10893 + operation: insert + start: 10000 + 31: + end: 5755 + operation: delete + start: 5000 + 32: + ids_end: 9000 + ids_start: 8773 + operation: replace + tags_end: 8227 + tags_start: 8000 + 33: + operation: search + 34: + end: 11612 + operation: insert + start: 11000 + 35: + end: 6604 + operation: delete + start: 6000 + 36: + ids_end: 10000 + ids_start: 9821 + operation: replace + tags_end: 9179 + tags_start: 9000 + 37: + operation: search + 38: + end: 12609 + operation: insert + start: 12000 + 39: + end: 7644 + operation: delete + start: 7000 + 40: + ids_end: 11000 + ids_start: 10893 + operation: replace + tags_end: 10107 + tags_start: 10000 + 41: + operation: search + 42: + end: 13825 + operation: insert + start: 13000 + 43: + end: 8773 + operation: delete + start: 8000 + 44: + ids_end: 12000 + ids_start: 11612 + operation: replace + tags_end: 11388 + tags_start: 11000 + 45: + operation: search + 46: + end: 14698 + operation: insert + start: 14000 + 47: + end: 9821 + operation: delete + start: 9000 + 48: + ids_end: 13000 + ids_start: 12609 + operation: replace + tags_end: 12391 + tags_start: 12000 + 49: + operation: search + 50: + end: 15507 + operation: insert + start: 15000 + 51: + end: 10893 + operation: delete + start: 10000 + 52: + ids_end: 14000 + ids_start: 13825 + operation: replace + tags_end: 13175 + tags_start: 13000 + 53: + operation: search + 54: + end: 16541 + operation: insert + start: 16000 + 55: + end: 11612 + operation: delete + start: 11000 + 56: + ids_end: 15000 + ids_start: 14698 + operation: replace + tags_end: 14302 + tags_start: 14000 + 57: + operation: search + 58: + end: 17549 + operation: insert + start: 17000 + 59: + end: 12609 + operation: delete + start: 12000 + 60: + ids_end: 16000 + ids_start: 15507 + operation: replace + tags_end: 15493 + tags_start: 15000 + 61: + operation: search + 62: + end: 18605 + operation: insert + start: 18000 + 63: + end: 13825 + operation: delete + start: 13000 + 64: + ids_end: 17000 + ids_start: 16541 + operation: replace + tags_end: 16459 + tags_start: 16000 + 65: + operation: search + 66: + end: 19600 + operation: insert + start: 19000 + 67: + end: 14698 + operation: delete + start: 14000 + 68: + operation: search + 69: + end: 20816 + operation: insert + start: 20000 + 70: + end: 15507 + operation: delete + start: 15000 + 71: + ids_end: 19000 + ids_start: 18605 + operation: replace + tags_end: 18395 + tags_start: 18000 + 72: + operation: search + 73: + end: 21853 + operation: insert + start: 21000 + 74: + end: 16541 + operation: delete + start: 16000 + 75: + ids_end: 20000 + ids_start: 19600 + operation: replace + tags_end: 19400 + tags_start: 19000 + 76: + operation: search + 77: + end: 22719 + operation: insert + start: 22000 + 78: + ids_end: 21000 + ids_start: 20816 + operation: replace + tags_end: 20184 + tags_start: 20000 + 79: + operation: search + 80: + end: 23610 + operation: insert + start: 23000 + 81: + end: 18605 + operation: delete + start: 18000 + 82: + operation: search + 83: + end: 24830 + operation: insert + start: 24000 + 84: + end: 19600 + operation: delete + start: 19000 + 85: + ids_end: 23000 + ids_start: 22719 + operation: replace + tags_end: 22281 + tags_start: 22000 + 86: + operation: search + 87: + end: 25742 + operation: insert + start: 25000 + 88: + end: 20816 + operation: delete + start: 20000 + 89: + ids_end: 24000 + ids_start: 23610 + operation: replace + tags_end: 23390 + tags_start: 23000 + 90: + operation: search + 91: + end: 26727 + operation: insert + start: 26000 + 92: + ids_end: 2000 + ids_start: 1777 + operation: replace + tags_end: 1223 + tags_start: 1000 + 93: + ids_end: 25000 + ids_start: 24830 + operation: replace + tags_end: 24170 + tags_start: 24000 + 94: + operation: search + 95: + end: 27858 + operation: insert + start: 27000 + 96: + end: 22719 + operation: delete + start: 22000 + 97: + operation: search + 98: + end: 28766 + operation: insert + start: 28000 + 99: + end: 23610 + operation: delete + start: 23000 + 100: + ids_end: 27000 + ids_start: 26727 + operation: replace + tags_end: 26273 + tags_start: 26000 + 101: + ids_end: 29000 + ids_start: 28766 + operation: replace + tags_end: 28234 + tags_start: 28000 + 102: + operation: search + 103: + end: 29514 + operation: insert + start: 29000 + 104: + end: 24830 + operation: delete + start: 24000 + 105: + ids_end: 28000 + ids_start: 27858 + operation: replace + tags_end: 27142 + tags_start: 27000 + 106: + operation: search + 107: + end: 30612 + operation: insert + start: 30000 + 108: + operation: search + 109: + end: 31840 + operation: insert + start: 31000 + 110: + end: 26727 + operation: delete + start: 26000 + 111: + ids_end: 26000 + ids_start: 25742 + operation: replace + tags_end: 25258 + tags_start: 25000 + 112: + ids_end: 30000 + ids_start: 29514 + operation: replace + tags_end: 29486 + tags_start: 29000 + 113: + operation: search + 114: + end: 32766 + operation: insert + start: 32000 + 115: + end: 27858 + operation: delete + start: 27000 + 116: + operation: search + 117: + end: 33534 + operation: insert + start: 33000 + 118: + ids_end: 32000 + ids_start: 31840 + operation: replace + tags_end: 31160 + tags_start: 31000 + 119: + operation: search + 120: + end: 34710 + operation: insert + start: 34000 + 121: + end: 29514 + operation: delete + start: 29000 + 122: + operation: search + 123: + end: 35894 + operation: insert + start: 35000 + 124: + ids_end: 34000 + ids_start: 33534 + operation: replace + tags_end: 33466 + tags_start: 33000 + 125: + operation: search + 126: + end: 36556 + operation: insert + start: 36000 + 127: + end: 31840 + operation: delete + start: 31000 + 128: + ids_end: 35000 + ids_start: 34710 + operation: replace + tags_end: 34290 + tags_start: 34000 + 129: + operation: search + 130: + end: 37513 + operation: insert + start: 37000 + 131: + ids_end: 36000 + ids_start: 35894 + operation: replace + tags_end: 35106 + tags_start: 35000 + 132: + operation: search + 133: + end: 38701 + operation: insert + start: 38000 + 134: + end: 33534 + operation: delete + start: 33000 + 135: + ids_end: 37000 + ids_start: 36556 + operation: replace + tags_end: 36444 + tags_start: 36000 + 136: + operation: search + 137: + end: 39815 + operation: insert + start: 39000 + 138: + end: 34710 + operation: delete + start: 34000 + 139: + ids_end: 38000 + ids_start: 37513 + operation: replace + tags_end: 37487 + tags_start: 37000 + 140: + operation: search + 141: + end: 40558 + operation: insert + start: 40000 + 142: + end: 35894 + operation: delete + start: 35000 + 143: + ids_end: 39000 + ids_start: 38701 + operation: replace + tags_end: 38299 + tags_start: 38000 + 144: + operation: search + 145: + end: 41839 + operation: insert + start: 41000 + 146: + end: 36556 + operation: delete + start: 36000 + 147: + operation: search + 148: + end: 42601 + operation: insert + start: 42000 + 149: + end: 37513 + operation: delete + start: 37000 + 150: + ids_end: 18000 + ids_start: 17549 + operation: replace + tags_end: 17451 + tags_start: 17000 + 151: + ids_end: 41000 + ids_start: 40558 + operation: replace + tags_end: 40442 + tags_start: 40000 + 152: + operation: search + 153: + end: 43598 + operation: insert + start: 43000 + 154: + end: 38701 + operation: delete + start: 38000 + 155: + ids_end: 42000 + ids_start: 41839 + operation: replace + tags_end: 41161 + tags_start: 41000 + 156: + operation: search + 157: + end: 44848 + operation: insert + start: 44000 + 158: + operation: search + 159: + end: 45558 + operation: insert + start: 45000 + 160: + end: 40558 + operation: delete + start: 40000 + 161: + ids_end: 44000 + ids_start: 43598 + operation: replace + tags_end: 43402 + tags_start: 43000 + 162: + operation: search + 163: + end: 46509 + operation: insert + start: 46000 + 164: + end: 41839 + operation: delete + start: 41000 + 165: + ids_end: 22000 + ids_start: 21853 + operation: replace + tags_end: 21147 + tags_start: 21000 + 166: + ids_end: 45000 + ids_start: 44848 + operation: replace + tags_end: 44152 + tags_start: 44000 + 167: + operation: search + 168: + end: 47899 + operation: insert + start: 47000 + 169: + ids_end: 46000 + ids_start: 45558 + operation: replace + tags_end: 45442 + tags_start: 45000 + 170: + operation: search + 171: + end: 48603 + operation: insert + start: 48000 + 172: + end: 43598 + operation: delete + start: 43000 + 173: + ids_end: 47000 + ids_start: 46509 + operation: replace + tags_end: 46491 + tags_start: 46000 + 174: + operation: search + 175: + end: 49892 + operation: insert + start: 49000 + 176: + end: 44848 + operation: delete + start: 44000 + 177: + ids_end: 1000 + ids_start: 834 + operation: replace + tags_end: 166 + tags_start: 0 + 178: + ids_end: 48000 + ids_start: 47899 + operation: replace + tags_end: 47101 + tags_start: 47000 + 179: + operation: search + 180: + end: 50887 + operation: insert + start: 50000 + 181: + end: 45558 + operation: delete + start: 45000 + 182: + operation: search + 183: + end: 51510 + operation: insert + start: 51000 + 184: + end: 1777 + operation: delete + start: 1000 + 185: + end: 46509 + operation: delete + start: 46000 + 186: + ids_end: 50000 + ids_start: 49892 + operation: replace + tags_end: 49108 + tags_start: 49000 + 187: + operation: search + 188: + end: 52877 + operation: insert + start: 52000 + 189: + end: 47899 + operation: delete + start: 47000 + 190: + ids_end: 51000 + ids_start: 50887 + operation: replace + tags_end: 50113 + tags_start: 50000 + 191: + operation: search + 192: + end: 53504 + operation: insert + start: 53000 + 193: + ids_end: 52000 + ids_start: 51510 + operation: replace + tags_end: 51490 + tags_start: 51000 + 194: + operation: search + 195: + end: 54682 + operation: insert + start: 54000 + 196: + end: 49892 + operation: delete + start: 49000 + 197: + ids_end: 53000 + ids_start: 52877 + operation: replace + tags_end: 52123 + tags_start: 52000 + 198: + operation: search + 199: + end: 55606 + operation: insert + start: 55000 + 200: + end: 50887 + operation: delete + start: 50000 + 201: + ids_end: 31000 + ids_start: 30612 + operation: replace + tags_end: 30388 + tags_start: 30000 + 202: + operation: search + 203: + end: 56888 + operation: insert + start: 56000 + 204: + end: 51510 + operation: delete + start: 51000 + 205: + ids_end: 55000 + ids_start: 54682 + operation: replace + tags_end: 54318 + tags_start: 54000 + 206: + operation: search + 207: + end: 57551 + operation: insert + start: 57000 + 208: + end: 52877 + operation: delete + start: 52000 + 209: + ids_end: 56000 + ids_start: 55606 + operation: replace + tags_end: 55394 + tags_start: 55000 + 210: + operation: search + 211: + end: 58814 + operation: insert + start: 58000 + 212: + ids_end: 57000 + ids_start: 56888 + operation: replace + tags_end: 56112 + tags_start: 56000 + 213: + operation: search + 214: + end: 59546 + operation: insert + start: 59000 + 215: + end: 54682 + operation: delete + start: 54000 + 216: + operation: search + 217: + end: 60857 + operation: insert + start: 60000 + 218: + end: 55606 + operation: delete + start: 55000 + 219: + operation: search + 220: + end: 61805 + operation: insert + start: 61000 + 221: + end: 56888 + operation: delete + start: 56000 + 222: + ids_end: 60000 + ids_start: 59546 + operation: replace + tags_end: 59454 + tags_start: 59000 + 223: + operation: search + 224: + end: 62587 + operation: insert + start: 62000 + 225: + operation: search + 226: + end: 63650 + operation: insert + start: 63000 + 227: + ids_end: 62000 + ids_start: 61805 + operation: replace + tags_end: 61195 + tags_start: 61000 + 228: + operation: search + 229: + end: 64699 + operation: insert + start: 64000 + 230: + end: 59546 + operation: delete + start: 59000 + 231: + ids_end: 40000 + ids_start: 39815 + operation: replace + tags_end: 39185 + tags_start: 39000 + 232: + ids_end: 63000 + ids_start: 62587 + operation: replace + tags_end: 62413 + tags_start: 62000 + 233: + operation: search + 234: + end: 65641 + operation: insert + start: 65000 + 235: + operation: search + 236: + end: 66860 + operation: insert + start: 66000 + 237: + end: 61805 + operation: delete + start: 61000 + 238: + operation: search + 239: + end: 67809 + operation: insert + start: 67000 + 240: + end: 17549 + operation: delete + start: 17000 + 241: + end: 62587 + operation: delete + start: 62000 + 242: + ids_end: 43000 + ids_start: 42601 + operation: replace + tags_end: 42399 + tags_start: 42000 + 243: + operation: search + 244: + end: 68679 + operation: insert + start: 68000 + 245: + ids_end: 67000 + ids_start: 66860 + operation: replace + tags_end: 66140 + tags_start: 66000 + 246: + operation: search + 247: + end: 69682 + operation: insert + start: 69000 + 248: + ids_end: 68000 + ids_start: 67809 + operation: replace + tags_end: 67191 + tags_start: 67000 + 249: + operation: search + 250: + end: 70659 + operation: insert + start: 70000 + 251: + ids_end: 69000 + ids_start: 68679 + operation: replace + tags_end: 68321 + tags_start: 68000 + 252: + operation: search + 253: + end: 71660 + operation: insert + start: 71000 + 254: + end: 21853 + operation: delete + start: 21000 + 255: + end: 66860 + operation: delete + start: 66000 + 256: + ids_end: 70000 + ids_start: 69682 + operation: replace + tags_end: 69318 + tags_start: 69000 + 257: + operation: search + 258: + end: 72760 + operation: insert + start: 72000 + 259: + end: 67809 + operation: delete + start: 67000 + 260: + ids_end: 71000 + ids_start: 70659 + operation: replace + tags_end: 70341 + tags_start: 70000 + 261: + operation: search + 262: + end: 73655 + operation: insert + start: 73000 + 263: + end: 68679 + operation: delete + start: 68000 + 264: + ids_end: 49000 + ids_start: 48603 + operation: replace + tags_end: 48397 + tags_start: 48000 + 265: + ids_end: 72000 + ids_start: 71660 + operation: replace + tags_end: 71340 + tags_start: 71000 + 266: + operation: search + 267: + end: 74665 + operation: insert + start: 74000 + 268: + end: 69682 + operation: delete + start: 69000 + 269: + ids_end: 73000 + ids_start: 72760 + operation: replace + tags_end: 72240 + tags_start: 72000 + 270: + operation: search + 271: + end: 75801 + operation: insert + start: 75000 + 272: + end: 70659 + operation: delete + start: 70000 + 273: + ids_end: 74000 + ids_start: 73655 + operation: replace + tags_end: 73345 + tags_start: 73000 + 274: + operation: search + 275: + end: 76524 + operation: insert + start: 76000 + 276: + end: 71660 + operation: delete + start: 71000 + 277: + ids_end: 75000 + ids_start: 74665 + operation: replace + tags_end: 74335 + tags_start: 74000 + 278: + operation: search + 279: + end: 77827 + operation: insert + start: 77000 + 280: + end: 72760 + operation: delete + start: 72000 + 281: + operation: search + 282: + end: 78639 + operation: insert + start: 78000 + 283: + end: 73655 + operation: delete + start: 73000 + 284: + ids_end: 54000 + ids_start: 53504 + operation: replace + tags_end: 53496 + tags_start: 53000 + 285: + ids_end: 77000 + ids_start: 76524 + operation: replace + tags_end: 76476 + tags_start: 76000 + 286: + operation: search + 287: + end: 79739 + operation: insert + start: 79000 + 288: + end: 74665 + operation: delete + start: 74000 + 289: + ids_end: 78000 + ids_start: 77827 + operation: replace + tags_end: 77173 + tags_start: 77000 + 290: + operation: search + 291: + end: 80611 + operation: insert + start: 80000 + 292: + end: 30612 + operation: delete + start: 30000 + 293: + ids_end: 79000 + ids_start: 78639 + operation: replace + tags_end: 78361 + tags_start: 78000 + 294: + operation: search + 295: + end: 81818 + operation: insert + start: 81000 + 296: + end: 76524 + operation: delete + start: 76000 + 297: + ids_end: 80000 + ids_start: 79739 + operation: replace + tags_end: 79261 + tags_start: 79000 + 298: + operation: search + 299: + end: 82668 + operation: insert + start: 82000 + 300: + end: 77827 + operation: delete + start: 77000 + 301: + ids_end: 58000 + ids_start: 57551 + operation: replace + tags_end: 57449 + tags_start: 57000 + 302: + ids_end: 81000 + ids_start: 80611 + operation: replace + tags_end: 80389 + tags_start: 80000 + 303: + operation: search + 304: + end: 83790 + operation: insert + start: 83000 + 305: + end: 78639 + operation: delete + start: 78000 + 306: + ids_end: 59000 + ids_start: 58814 + operation: replace + tags_end: 58186 + tags_start: 58000 + 307: + ids_end: 82000 + ids_start: 81818 + operation: replace + tags_end: 81182 + tags_start: 81000 + 308: + operation: search + 309: + end: 84734 + operation: insert + start: 84000 + 310: + end: 79739 + operation: delete + start: 79000 + 311: + ids_end: 83000 + ids_start: 82668 + operation: replace + tags_end: 82332 + tags_start: 82000 + 312: + operation: search + 313: + end: 85640 + operation: insert + start: 85000 + 314: + end: 80611 + operation: delete + start: 80000 + 315: + ids_end: 84000 + ids_start: 83790 + operation: replace + tags_end: 83210 + tags_start: 83000 + 316: + operation: search + 317: + end: 86575 + operation: insert + start: 86000 + 318: + end: 81818 + operation: delete + start: 81000 + 319: + ids_end: 85000 + ids_start: 84734 + operation: replace + tags_end: 84266 + tags_start: 84000 + 320: + operation: search + 321: + end: 87723 + operation: insert + start: 87000 + 322: + end: 82668 + operation: delete + start: 82000 + 323: + ids_end: 86000 + ids_start: 85640 + operation: replace + tags_end: 85360 + tags_start: 85000 + 324: + operation: search + 325: + end: 88731 + operation: insert + start: 88000 + 326: + end: 83790 + operation: delete + start: 83000 + 327: + ids_end: 87000 + ids_start: 86575 + operation: replace + tags_end: 86425 + tags_start: 86000 + 328: + operation: search + 329: + end: 89722 + operation: insert + start: 89000 + 330: + end: 39815 + operation: delete + start: 39000 + 331: + end: 84734 + operation: delete + start: 84000 + 332: + ids_end: 65000 + ids_start: 64699 + operation: replace + tags_end: 64301 + tags_start: 64000 + 333: + operation: search + 334: + end: 90649 + operation: insert + start: 90000 + 335: + end: 85640 + operation: delete + start: 85000 + 336: + ids_end: 66000 + ids_start: 65641 + operation: replace + tags_end: 65359 + tags_start: 65000 + 337: + ids_end: 89000 + ids_start: 88731 + operation: replace + tags_end: 88269 + tags_start: 88000 + 338: + operation: search + 339: + end: 91510 + operation: insert + start: 91000 + 340: + end: 86575 + operation: delete + start: 86000 + 341: + ids_end: 90000 + ids_start: 89722 + operation: replace + tags_end: 89278 + tags_start: 89000 + 342: + operation: search + 343: + end: 92554 + operation: insert + start: 92000 + 344: + end: 42601 + operation: delete + start: 42000 + 345: + ids_end: 91000 + ids_start: 90649 + operation: replace + tags_end: 90351 + tags_start: 90000 + 346: + operation: search + 347: + end: 93672 + operation: insert + start: 93000 + 348: + end: 88731 + operation: delete + start: 88000 + 349: + ids_end: 92000 + ids_start: 91510 + operation: replace + tags_end: 91490 + tags_start: 91000 + 350: + operation: search + 351: + end: 94540 + operation: insert + start: 94000 + 352: + end: 89722 + operation: delete + start: 89000 + 353: + ids_end: 93000 + ids_start: 92554 + operation: replace + tags_end: 92446 + tags_start: 92000 + 354: + operation: search + 355: + end: 95576 + operation: insert + start: 95000 + 356: + end: 90649 + operation: delete + start: 90000 + 357: + ids_end: 94000 + ids_start: 93672 + operation: replace + tags_end: 93328 + tags_start: 93000 + 358: + operation: search + 359: + end: 96606 + operation: insert + start: 96000 + 360: + end: 91510 + operation: delete + start: 91000 + 361: + ids_end: 95000 + ids_start: 94540 + operation: replace + tags_end: 94460 + tags_start: 94000 + 362: + operation: search + 363: + end: 97733 + operation: insert + start: 97000 + 364: + end: 92554 + operation: delete + start: 92000 + 365: + ids_end: 96000 + ids_start: 95576 + operation: replace + tags_end: 95424 + tags_start: 95000 + 366: + operation: search + 367: + end: 98640 + operation: insert + start: 98000 + 368: + end: 48603 + operation: delete + start: 48000 + 369: + end: 93672 + operation: delete + start: 93000 + 370: + operation: search + 371: + end: 99550 + operation: insert + start: 99000 + 372: + end: 94540 + operation: delete + start: 94000 + 373: + ids_end: 64000 + ids_start: 63650 + operation: replace + tags_end: 63350 + tags_start: 63000 + 374: + ids_end: 98000 + ids_start: 97733 + operation: replace + tags_end: 97267 + tags_start: 97000 + 375: + operation: search + max_pts: 5785 + gt_url: https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturing-100K_expiration_time_replace_delete_runbook.yaml diff --git a/neurips23/runbooks/msturingpq-100K_expiration_time_replace_only_runbook.yaml b/neurips23/runbooks/msturingpq-100K_expiration_time_replace_only_runbook.yaml new file mode 100644 index 00000000..86f9c8c4 --- /dev/null +++ b/neurips23/runbooks/msturingpq-100K_expiration_time_replace_only_runbook.yaml @@ -0,0 +1,1161 @@ +msturingpq-100K: + 1: + end: 834 + operation: insert + start: 0 + 2: + operation: search + 3: + end: 1777 + operation: insert + start: 1000 + 4: + operation: search + 5: + end: 2772 + operation: insert + start: 2000 + 6: + operation: search + 7: + end: 3664 + operation: insert + start: 3000 + 8: + operation: search + 9: + end: 4735 + operation: insert + start: 4000 + 10: + ids_end: 3000 + ids_start: 2772 + operation: replace + tags_end: 2228 + tags_start: 2000 + 11: + operation: search + 12: + end: 5755 + operation: insert + start: 5000 + 13: + ids_end: 4000 + ids_start: 3664 + operation: replace + tags_end: 3336 + tags_start: 3000 + 14: + operation: search + 15: + end: 6604 + operation: insert + start: 6000 + 16: + ids_end: 5000 + ids_start: 4735 + operation: replace + tags_end: 4265 + tags_start: 4000 + 17: + operation: search + 18: + end: 7644 + operation: insert + start: 7000 + 19: + ids_end: 6000 + ids_start: 5755 + operation: replace + tags_end: 5245 + tags_start: 5000 + 20: + operation: search + 21: + end: 8773 + operation: insert + start: 8000 + 22: + ids_end: 7000 + ids_start: 6604 + operation: replace + tags_end: 6396 + tags_start: 6000 + 23: + operation: search + 24: + end: 9821 + operation: insert + start: 9000 + 25: + ids_end: 8000 + ids_start: 7644 + operation: replace + tags_end: 7356 + tags_start: 7000 + 26: + operation: search + 27: + end: 10893 + operation: insert + start: 10000 + 28: + ids_end: 9000 + ids_start: 8773 + operation: replace + tags_end: 8227 + tags_start: 8000 + 29: + operation: search + 30: + end: 11612 + operation: insert + start: 11000 + 31: + ids_end: 10000 + ids_start: 9821 + operation: replace + tags_end: 9179 + tags_start: 9000 + 32: + operation: search + 33: + end: 12609 + operation: insert + start: 12000 + 34: + ids_end: 11000 + ids_start: 10893 + operation: replace + tags_end: 10107 + tags_start: 10000 + 35: + operation: search + 36: + end: 13825 + operation: insert + start: 13000 + 37: + ids_end: 12000 + ids_start: 11612 + operation: replace + tags_end: 11388 + tags_start: 11000 + 38: + operation: search + 39: + end: 14698 + operation: insert + start: 14000 + 40: + ids_end: 13000 + ids_start: 12609 + operation: replace + tags_end: 12391 + tags_start: 12000 + 41: + operation: search + 42: + end: 15507 + operation: insert + start: 15000 + 43: + ids_end: 14000 + ids_start: 13825 + operation: replace + tags_end: 13175 + tags_start: 13000 + 44: + operation: search + 45: + end: 16541 + operation: insert + start: 16000 + 46: + ids_end: 15000 + ids_start: 14698 + operation: replace + tags_end: 14302 + tags_start: 14000 + 47: + operation: search + 48: + end: 17549 + operation: insert + start: 17000 + 49: + ids_end: 16000 + ids_start: 15507 + operation: replace + tags_end: 15493 + tags_start: 15000 + 50: + operation: search + 51: + end: 18605 + operation: insert + start: 18000 + 52: + ids_end: 17000 + ids_start: 16541 + operation: replace + tags_end: 16459 + tags_start: 16000 + 53: + operation: search + 54: + end: 19600 + operation: insert + start: 19000 + 55: + operation: search + 56: + end: 20816 + operation: insert + start: 20000 + 57: + ids_end: 19000 + ids_start: 18605 + operation: replace + tags_end: 18395 + tags_start: 18000 + 58: + operation: search + 59: + end: 21853 + operation: insert + start: 21000 + 60: + ids_end: 20000 + ids_start: 19600 + operation: replace + tags_end: 19400 + tags_start: 19000 + 61: + operation: search + 62: + end: 22719 + operation: insert + start: 22000 + 63: + ids_end: 21000 + ids_start: 20816 + operation: replace + tags_end: 20184 + tags_start: 20000 + 64: + operation: search + 65: + end: 23610 + operation: insert + start: 23000 + 66: + operation: search + 67: + end: 24830 + operation: insert + start: 24000 + 68: + ids_end: 23000 + ids_start: 22719 + operation: replace + tags_end: 22281 + tags_start: 22000 + 69: + operation: search + 70: + end: 25742 + operation: insert + start: 25000 + 71: + ids_end: 24000 + ids_start: 23610 + operation: replace + tags_end: 23390 + tags_start: 23000 + 72: + operation: search + 73: + end: 26727 + operation: insert + start: 26000 + 74: + ids_end: 2000 + ids_start: 1777 + operation: replace + tags_end: 1223 + tags_start: 1000 + 75: + ids_end: 25000 + ids_start: 24830 + operation: replace + tags_end: 24170 + tags_start: 24000 + 76: + operation: search + 77: + end: 27858 + operation: insert + start: 27000 + 78: + operation: search + 79: + end: 28766 + operation: insert + start: 28000 + 80: + ids_end: 27000 + ids_start: 26727 + operation: replace + tags_end: 26273 + tags_start: 26000 + 81: + operation: search + 82: + end: 29514 + operation: insert + start: 29000 + 83: + ids_end: 28000 + ids_start: 27858 + operation: replace + tags_end: 27142 + tags_start: 27000 + 84: + operation: search + 85: + end: 30612 + operation: insert + start: 30000 + 86: + operation: search + 87: + end: 31840 + operation: insert + start: 31000 + 88: + ids_end: 30000 + ids_start: 29514 + operation: replace + tags_end: 29486 + tags_start: 29000 + 89: + operation: search + 90: + end: 32766 + operation: insert + start: 32000 + 91: + operation: search + 92: + end: 33534 + operation: insert + start: 33000 + 93: + ids_end: 32000 + ids_start: 31840 + operation: replace + tags_end: 31160 + tags_start: 31000 + 94: + operation: search + 95: + end: 34710 + operation: insert + start: 34000 + 96: + operation: search + 97: + end: 35894 + operation: insert + start: 35000 + 98: + ids_end: 34000 + ids_start: 33534 + operation: replace + tags_end: 33466 + tags_start: 33000 + 99: + operation: search + 100: + end: 36556 + operation: insert + start: 36000 + 101: + ids_end: 33000 + ids_start: 32766 + operation: replace + tags_end: 32234 + tags_start: 32000 + 102: + ids_end: 35000 + ids_start: 34710 + operation: replace + tags_end: 34290 + tags_start: 34000 + 103: + operation: search + 104: + end: 37513 + operation: insert + start: 37000 + 105: + ids_end: 36000 + ids_start: 35894 + operation: replace + tags_end: 35106 + tags_start: 35000 + 106: + operation: search + 107: + end: 38701 + operation: insert + start: 38000 + 108: + ids_end: 29000 + ids_start: 28766 + operation: replace + tags_end: 28234 + tags_start: 28000 + 109: + ids_end: 37000 + ids_start: 36556 + operation: replace + tags_end: 36444 + tags_start: 36000 + 110: + operation: search + 111: + end: 39815 + operation: insert + start: 39000 + 112: + ids_end: 26000 + ids_start: 25742 + operation: replace + tags_end: 25258 + tags_start: 25000 + 113: + ids_end: 38000 + ids_start: 37513 + operation: replace + tags_end: 37487 + tags_start: 37000 + 114: + operation: search + 115: + end: 40558 + operation: insert + start: 40000 + 116: + ids_end: 39000 + ids_start: 38701 + operation: replace + tags_end: 38299 + tags_start: 38000 + 117: + operation: search + 118: + end: 41839 + operation: insert + start: 41000 + 119: + operation: search + 120: + end: 42601 + operation: insert + start: 42000 + 121: + ids_end: 18000 + ids_start: 17549 + operation: replace + tags_end: 17451 + tags_start: 17000 + 122: + ids_end: 41000 + ids_start: 40558 + operation: replace + tags_end: 40442 + tags_start: 40000 + 123: + operation: search + 124: + end: 43598 + operation: insert + start: 43000 + 125: + ids_end: 42000 + ids_start: 41839 + operation: replace + tags_end: 41161 + tags_start: 41000 + 126: + operation: search + 127: + end: 44848 + operation: insert + start: 44000 + 128: + operation: search + 129: + end: 45558 + operation: insert + start: 45000 + 130: + ids_end: 44000 + ids_start: 43598 + operation: replace + tags_end: 43402 + tags_start: 43000 + 131: + operation: search + 132: + end: 46509 + operation: insert + start: 46000 + 133: + ids_end: 22000 + ids_start: 21853 + operation: replace + tags_end: 21147 + tags_start: 21000 + 134: + ids_end: 45000 + ids_start: 44848 + operation: replace + tags_end: 44152 + tags_start: 44000 + 135: + operation: search + 136: + end: 47899 + operation: insert + start: 47000 + 137: + ids_end: 46000 + ids_start: 45558 + operation: replace + tags_end: 45442 + tags_start: 45000 + 138: + operation: search + 139: + end: 48603 + operation: insert + start: 48000 + 140: + ids_end: 47000 + ids_start: 46509 + operation: replace + tags_end: 46491 + tags_start: 46000 + 141: + operation: search + 142: + end: 49892 + operation: insert + start: 49000 + 143: + ids_end: 1000 + ids_start: 834 + operation: replace + tags_end: 166 + tags_start: 0 + 144: + ids_end: 48000 + ids_start: 47899 + operation: replace + tags_end: 47101 + tags_start: 47000 + 145: + operation: search + 146: + end: 50887 + operation: insert + start: 50000 + 147: + operation: search + 148: + end: 51510 + operation: insert + start: 51000 + 149: + ids_end: 50000 + ids_start: 49892 + operation: replace + tags_end: 49108 + tags_start: 49000 + 150: + operation: search + 151: + end: 52877 + operation: insert + start: 52000 + 152: + ids_end: 51000 + ids_start: 50887 + operation: replace + tags_end: 50113 + tags_start: 50000 + 153: + operation: search + 154: + end: 53504 + operation: insert + start: 53000 + 155: + ids_end: 52000 + ids_start: 51510 + operation: replace + tags_end: 51490 + tags_start: 51000 + 156: + operation: search + 157: + end: 54682 + operation: insert + start: 54000 + 158: + ids_end: 53000 + ids_start: 52877 + operation: replace + tags_end: 52123 + tags_start: 52000 + 159: + operation: search + 160: + end: 55606 + operation: insert + start: 55000 + 161: + ids_end: 31000 + ids_start: 30612 + operation: replace + tags_end: 30388 + tags_start: 30000 + 162: + operation: search + 163: + end: 56888 + operation: insert + start: 56000 + 164: + ids_end: 55000 + ids_start: 54682 + operation: replace + tags_end: 54318 + tags_start: 54000 + 165: + operation: search + 166: + end: 57551 + operation: insert + start: 57000 + 167: + ids_end: 56000 + ids_start: 55606 + operation: replace + tags_end: 55394 + tags_start: 55000 + 168: + operation: search + 169: + end: 58814 + operation: insert + start: 58000 + 170: + ids_end: 57000 + ids_start: 56888 + operation: replace + tags_end: 56112 + tags_start: 56000 + 171: + operation: search + 172: + end: 59546 + operation: insert + start: 59000 + 173: + operation: search + 174: + end: 60857 + operation: insert + start: 60000 + 175: + operation: search + 176: + end: 61805 + operation: insert + start: 61000 + 177: + ids_end: 60000 + ids_start: 59546 + operation: replace + tags_end: 59454 + tags_start: 59000 + 178: + operation: search + 179: + end: 62587 + operation: insert + start: 62000 + 180: + operation: search + 181: + end: 63650 + operation: insert + start: 63000 + 182: + ids_end: 62000 + ids_start: 61805 + operation: replace + tags_end: 61195 + tags_start: 61000 + 183: + operation: search + 184: + end: 64699 + operation: insert + start: 64000 + 185: + ids_end: 40000 + ids_start: 39815 + operation: replace + tags_end: 39185 + tags_start: 39000 + 186: + ids_end: 63000 + ids_start: 62587 + operation: replace + tags_end: 62413 + tags_start: 62000 + 187: + operation: search + 188: + end: 65641 + operation: insert + start: 65000 + 189: + operation: search + 190: + end: 66860 + operation: insert + start: 66000 + 191: + operation: search + 192: + end: 67809 + operation: insert + start: 67000 + 193: + ids_end: 43000 + ids_start: 42601 + operation: replace + tags_end: 42399 + tags_start: 42000 + 194: + operation: search + 195: + end: 68679 + operation: insert + start: 68000 + 196: + ids_end: 67000 + ids_start: 66860 + operation: replace + tags_end: 66140 + tags_start: 66000 + 197: + operation: search + 198: + end: 69682 + operation: insert + start: 69000 + 199: + ids_end: 68000 + ids_start: 67809 + operation: replace + tags_end: 67191 + tags_start: 67000 + 200: + operation: search + 201: + end: 70659 + operation: insert + start: 70000 + 202: + ids_end: 69000 + ids_start: 68679 + operation: replace + tags_end: 68321 + tags_start: 68000 + 203: + operation: search + 204: + end: 71660 + operation: insert + start: 71000 + 205: + ids_end: 70000 + ids_start: 69682 + operation: replace + tags_end: 69318 + tags_start: 69000 + 206: + operation: search + 207: + end: 72760 + operation: insert + start: 72000 + 208: + ids_end: 71000 + ids_start: 70659 + operation: replace + tags_end: 70341 + tags_start: 70000 + 209: + operation: search + 210: + end: 73655 + operation: insert + start: 73000 + 211: + ids_end: 49000 + ids_start: 48603 + operation: replace + tags_end: 48397 + tags_start: 48000 + 212: + ids_end: 72000 + ids_start: 71660 + operation: replace + tags_end: 71340 + tags_start: 71000 + 213: + operation: search + 214: + end: 74665 + operation: insert + start: 74000 + 215: + ids_end: 73000 + ids_start: 72760 + operation: replace + tags_end: 72240 + tags_start: 72000 + 216: + operation: search + 217: + end: 75801 + operation: insert + start: 75000 + 218: + ids_end: 74000 + ids_start: 73655 + operation: replace + tags_end: 73345 + tags_start: 73000 + 219: + operation: search + 220: + end: 76524 + operation: insert + start: 76000 + 221: + ids_end: 75000 + ids_start: 74665 + operation: replace + tags_end: 74335 + tags_start: 74000 + 222: + operation: search + 223: + end: 77827 + operation: insert + start: 77000 + 224: + operation: search + 225: + end: 78639 + operation: insert + start: 78000 + 226: + ids_end: 54000 + ids_start: 53504 + operation: replace + tags_end: 53496 + tags_start: 53000 + 227: + ids_end: 77000 + ids_start: 76524 + operation: replace + tags_end: 76476 + tags_start: 76000 + 228: + operation: search + 229: + end: 79739 + operation: insert + start: 79000 + 230: + ids_end: 78000 + ids_start: 77827 + operation: replace + tags_end: 77173 + tags_start: 77000 + 231: + operation: search + 232: + end: 80611 + operation: insert + start: 80000 + 233: + ids_end: 79000 + ids_start: 78639 + operation: replace + tags_end: 78361 + tags_start: 78000 + 234: + operation: search + 235: + end: 81818 + operation: insert + start: 81000 + 236: + ids_end: 80000 + ids_start: 79739 + operation: replace + tags_end: 79261 + tags_start: 79000 + 237: + operation: search + 238: + end: 82668 + operation: insert + start: 82000 + 239: + ids_end: 58000 + ids_start: 57551 + operation: replace + tags_end: 57449 + tags_start: 57000 + 240: + ids_end: 81000 + ids_start: 80611 + operation: replace + tags_end: 80389 + tags_start: 80000 + 241: + operation: search + 242: + end: 83790 + operation: insert + start: 83000 + 243: + ids_end: 59000 + ids_start: 58814 + operation: replace + tags_end: 58186 + tags_start: 58000 + 244: + ids_end: 82000 + ids_start: 81818 + operation: replace + tags_end: 81182 + tags_start: 81000 + 245: + operation: search + 246: + end: 84734 + operation: insert + start: 84000 + 247: + ids_end: 83000 + ids_start: 82668 + operation: replace + tags_end: 82332 + tags_start: 82000 + 248: + operation: search + 249: + end: 85640 + operation: insert + start: 85000 + 250: + ids_end: 84000 + ids_start: 83790 + operation: replace + tags_end: 83210 + tags_start: 83000 + 251: + operation: search + 252: + end: 86575 + operation: insert + start: 86000 + 253: + ids_end: 85000 + ids_start: 84734 + operation: replace + tags_end: 84266 + tags_start: 84000 + 254: + operation: search + 255: + end: 87723 + operation: insert + start: 87000 + 256: + ids_end: 86000 + ids_start: 85640 + operation: replace + tags_end: 85360 + tags_start: 85000 + 257: + operation: search + 258: + end: 88731 + operation: insert + start: 88000 + 259: + ids_end: 87000 + ids_start: 86575 + operation: replace + tags_end: 86425 + tags_start: 86000 + 260: + operation: search + 261: + end: 89722 + operation: insert + start: 89000 + 262: + ids_end: 65000 + ids_start: 64699 + operation: replace + tags_end: 64301 + tags_start: 64000 + 263: + operation: search + 264: + end: 90649 + operation: insert + start: 90000 + 265: + ids_end: 66000 + ids_start: 65641 + operation: replace + tags_end: 65359 + tags_start: 65000 + 266: + ids_end: 89000 + ids_start: 88731 + operation: replace + tags_end: 88269 + tags_start: 88000 + 267: + operation: search + 268: + end: 91510 + operation: insert + start: 91000 + 269: + ids_end: 90000 + ids_start: 89722 + operation: replace + tags_end: 89278 + tags_start: 89000 + 270: + operation: search + 271: + end: 92554 + operation: insert + start: 92000 + 272: + ids_end: 91000 + ids_start: 90649 + operation: replace + tags_end: 90351 + tags_start: 90000 + 273: + operation: search + 274: + end: 93672 + operation: insert + start: 93000 + 275: + ids_end: 92000 + ids_start: 91510 + operation: replace + tags_end: 91490 + tags_start: 91000 + 276: + operation: search + 277: + end: 94540 + operation: insert + start: 94000 + 278: + ids_end: 93000 + ids_start: 92554 + operation: replace + tags_end: 92446 + tags_start: 92000 + 279: + operation: search + 280: + end: 95576 + operation: insert + start: 95000 + 281: + ids_end: 94000 + ids_start: 93672 + operation: replace + tags_end: 93328 + tags_start: 93000 + 282: + operation: search + 283: + end: 96606 + operation: insert + start: 96000 + 284: + ids_end: 95000 + ids_start: 94540 + operation: replace + tags_end: 94460 + tags_start: 94000 + 285: + operation: search + 286: + end: 97733 + operation: insert + start: 97000 + 287: + ids_end: 96000 + ids_start: 95576 + operation: replace + tags_end: 95424 + tags_start: 95000 + 288: + operation: search + 289: + end: 98640 + operation: insert + start: 98000 + 290: + operation: search + 291: + end: 99550 + operation: insert + start: 99000 + 292: + ids_end: 98000 + ids_start: 97733 + operation: replace + tags_end: 97267 + tags_start: 97000 + 293: + operation: search + max_pts: 69204 + gt_url: https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturing-100K_expiration_time_replace_only_runbook.yaml diff --git a/neurips23/runbooks/msturingpq-100K_expiration_time_runbook.yaml b/neurips23/runbooks/msturingpq-100K_expiration_time_runbook.yaml index 10d23071..f93c4583 100644 --- a/neurips23/runbooks/msturingpq-100K_expiration_time_runbook.yaml +++ b/neurips23/runbooks/msturingpq-100K_expiration_time_runbook.yaml @@ -1,4 +1,4 @@ -msturing-100K: +msturingpq-100K: 1: end: 1000 operation: insert @@ -915,5 +915,5 @@ msturing-100K: start: 94000 279: operation: search - max_pts: 28000 + max_pts: 24000 gt_url: https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturing-100K_expiration_time_runbook.yaml