diff --git a/apps/search_memory_index.cpp b/apps/search_memory_index.cpp index 1a9acc285..5dc1ea11f 100644 --- a/apps/search_memory_index.cpp +++ b/apps/search_memory_index.cpp @@ -109,9 +109,9 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path, } else { - std::cout << std::setw(4) << "Ls" << std::setw(12) << qps_title << std::setw(18) << "Avg dist cmps" + std::cout << std::setw(4) << "Ls" << std::setw(12) << qps_title << std::setw(18) << "Avg dist cmps" << std::setw(18) << "Avg hops" << std::setw(20) << "Mean Latency (mus)" << std::setw(15) << "99.9 Latency"; - table_width += 4 + 12 + 18 + 20 + 15; + table_width += 4 + 12 + 18 + 18+ 20 + 15; } uint32_t recalls_to_print = 0; const uint32_t first_recall = print_all_recalls ? 1 : recall_at; @@ -131,9 +131,11 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path, std::vector> query_result_dists(Lvec.size()); std::vector latency_stats(query_num, 0); std::vector cmp_stats; + std::vector hop_stats; if (not tags || filtered_search) { cmp_stats = std::vector(query_num, 0); + hop_stats = std::vector(query_num, 0); } std::vector query_result_tags; @@ -171,6 +173,7 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path, query_result_ids[test_id].data() + i * recall_at, query_result_dists[test_id].data() + i * recall_at); cmp_stats[i] = retval.second; + hop_stats[i] = retval.first; } else if (metric == diskann::FAST_L2) { @@ -203,6 +206,10 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path, ->search(query + i * query_aligned_dim, recall_at, L, query_result_ids[test_id].data() + i * recall_at) .second; + hop_stats[i] = index + ->search(query + i * query_aligned_dim, recall_at, L, + query_result_ids[test_id].data() + i * recall_at) + .first; } auto qe = std::chrono::high_resolution_clock::now(); std::chrono::duration diff = qe - qs; @@ -231,6 +238,7 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path, std::accumulate(latency_stats.begin(), latency_stats.end(), 0.0) / static_cast(query_num); float avg_cmps = (float)std::accumulate(cmp_stats.begin(), cmp_stats.end(), 0) / (float)query_num; + float avg_hops = (float)std::accumulate(hop_stats.begin(), hop_stats.end(), 0) / (float)query_num; if (tags && !filtered_search) { @@ -239,7 +247,7 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path, } else { - std::cout << std::setw(4) << L << std::setw(12) << displayed_qps << std::setw(18) << avg_cmps + std::cout << std::setw(4) << L << std::setw(12) << displayed_qps << std::setw(18) << avg_cmps << std::setw(18) << avg_hops << std::setw(20) << (float)mean_latency << std::setw(15) << (float)latency_stats[(uint64_t)(0.999 * query_num)]; } diff --git a/include/index.h b/include/index.h index b9bf4f384..a5326db4e 100644 --- a/include/index.h +++ b/include/index.h @@ -257,7 +257,7 @@ template clas // The query to use is placed in scratch->aligned_query std::pair iterate_to_fixed_point(InMemQueryScratch *scratch, const uint32_t Lindex, - const std::vector &init_ids, bool use_filter, + const size_t K, const std::vector &init_ids, bool use_filter, const std::vector &filters, bool search_invocation); void search_for_point_and_prune(int location, uint32_t Lindex, std::vector &pruned_list, diff --git a/include/neighbor.h b/include/neighbor.h index d7c0c25ed..d1b972c11 100644 --- a/include/neighbor.h +++ b/include/neighbor.h @@ -53,7 +53,7 @@ class NeighborPriorityQueue // item in the set. The set cursor that is used to pop() the // next item will be set to the lowest index of an uncheck item void insert(const Neighbor &nbr) - { + { if (_size == _capacity && _data[_size - 1] < nbr) { return; @@ -93,6 +93,36 @@ class NeighborPriorityQueue } } + int check_in_top_k(const Neighbor &nbr, size_t K) + { + if (_size == _capacity && _data[_size - 1] < nbr) + { + return 0; + } + + size_t lo = 0, hi = _size; + while (lo < hi) + { + size_t mid = (lo + hi) >> 1; + if (nbr < _data[mid]) + { + hi = mid; + } + else if (_data[mid].id == nbr.id) + { + return 0; + } + else + { + lo = mid + 1; + } + } + // if(lo < K){ + // return 1; + // } + return 1; + } + Neighbor closest_unexpanded() { _data[_cur].expanded = true; diff --git a/old_baseline/baseline_r64_k10.txt b/old_baseline/baseline_r64_k10.txt new file mode 100644 index 000000000..6b9af1d0e --- /dev/null +++ b/old_baseline/baseline_r64_k10.txt @@ -0,0 +1,84 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.40021s +From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 48 threads to search + Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@10 +================================================================================= + 10 51557.93 1324.67 913.18 9792.77 43.13 + 20 33640.81 2021.14 1420.26 3249.95 58.38 + 30 25210.59 2617.97 1896.59 4469.27 66.24 + 40 20547.19 3168.06 2326.82 5450.12 71.25 + 50 17328.33 3687.22 2760.19 6599.82 74.44 + 60 14980.50 4195.96 3193.32 7739.67 76.98 + 70 13341.03 4685.40 3588.47 8306.07 78.78 + 80 11462.47 5171.15 4176.58 11172.33 80.21 + 90 10723.69 5654.48 4462.36 10865.74 81.53 + 100 9782.57 6129.37 4892.21 12162.22 82.58 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_10_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_10_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_20_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_20_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_30_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_30_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_40_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_40_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. diff --git a/old_baseline/baseline_r64_k100.txt b/old_baseline/baseline_r64_k100.txt new file mode 100644 index 000000000..b6c634e16 --- /dev/null +++ b/old_baseline/baseline_r64_k100.txt @@ -0,0 +1,21 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.40809s +From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 48 threads to search + Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@100 +================================================================================= + 100 10331.70 6129.37 4626.05 11214.14 66.81 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 100, size = 5306008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 100, size = 5306008B +Finished writing bin. diff --git a/old_baseline/baseline_r64_k25.txt b/old_baseline/baseline_r64_k25.txt new file mode 100644 index 000000000..eaf63e5de --- /dev/null +++ b/old_baseline/baseline_r64_k25.txt @@ -0,0 +1,70 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.48129s +From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 48 threads to search + Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@25 +================================================================================= + 30 25253.53 2617.97 1885.01 10497.99 60.18 + 40 20654.60 3168.06 2316.21 5201.40 65.38 + 50 17413.17 3687.22 2747.47 6720.12 68.82 + 60 15010.45 4195.96 3187.87 7331.08 71.60 + 70 13300.10 4685.40 3599.56 8146.48 73.66 + 80 11990.02 5171.15 3991.75 8909.45 75.35 + 90 10918.86 5654.48 4385.05 9846.92 76.87 + 100 9976.54 6129.37 4799.29 10763.89 78.14 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_30_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_30_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_40_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_40_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. diff --git a/old_baseline/baseline_r64_k50.txt b/old_baseline/baseline_r64_k50.txt new file mode 100644 index 000000000..45e6b863f --- /dev/null +++ b/old_baseline/baseline_r64_k50.txt @@ -0,0 +1,56 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.39346s +From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@50 +================================================================================= + 50 750.71 3687.22 1332.03 15081.17 62.96 + 60 664.89 4195.96 1503.96 13729.61 65.92 + 70 595.61 4685.40 1678.89 12513.98 68.19 + 80 539.72 5171.15 1852.77 12737.32 70.07 + 90 493.28 5654.48 2027.21 12803.06 71.76 + 100 453.69 6129.37 2204.12 15871.16 73.17 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. diff --git a/old_baseline/baseline_r64_l100.txt b/old_baseline/baseline_r64_l100.txt new file mode 100644 index 000000000..b86d0c55f --- /dev/null +++ b/old_baseline/baseline_r64_l100.txt @@ -0,0 +1,10 @@ +Starting index build with R: 64 Lbuild: 100 alpha: 1.2 #threads: 48 +Using AVX2 distance computation DistanceL2Int8. +Using only first 40469609 from file.. +Starting index build with 40469609 points... + 0% of index build completed. 0.247099% of index build completed. 0.494198% of index build completed. 0.741297% of index build completed. 0.988396% of index build completed. 1.2355% of index build completed. 1.48259% of index build completed. 1.72969% of index build completed. 1.97679% of index build completed. 2.22389% of index build completed. 2.47099% of index build completed. 2.71809% of index build completed. 2.96519% of index build completed. 3.21229% of index build completed. 3.45939% of index build completed. 3.70649% of index build completed. 3.95358% of index build completed. 4.20068% of index build completed. 4.44778% of index build completed. 4.69488% of index build completed. 4.94198% of index build completed. 5.18908% of index build completed. 5.43618% of index build completed. 5.68328% of index build completed. 5.93038% of index build completed. 6.17748% of index build completed. 6.42457% of index build completed. 6.67167% of index build completed. 6.91877% of index build completed. 7.16587% of index build completed. 7.41297% of index build completed. 7.66007% of index build completed. 7.90717% of index build completed. 8.15427% of index build completed. 8.40137% of index build completed. 8.64847% of index build completed. 8.89556% of index build completed. 9.14266% of index build completed. 9.38976% of index build completed. 9.63686% of index build completed. 9.88396% of index build completed. 10.1311% of index build completed. 10.3782% of index build completed. 10.6253% of index build completed. 10.8724% of index build completed. 11.1195% of index build completed. 11.3666% of index build completed. 11.6137% of index build completed. 11.8608% of index build completed. 12.1079% of index build completed. 12.355% of index build completed. 12.602% of index build completed. 12.8491% of index build completed. 13.0962% of index build completed. 13.3433% of index build completed. 13.5904% of index build completed. 13.8375% of index build completed. 14.0846% of index build completed. 14.3317% of index build completed. 14.5788% of index build completed. 14.8259% of index build completed. 15.073% of index build completed. 15.3201% of index build completed. 15.5672% of index build completed. 15.8143% of index build completed. 16.0614% of index build completed. 16.3085% of index build completed. 16.5556% of index build completed. 16.8027% of index build completed. 17.0498% of index build completed. 17.2969% of index build completed. 17.544% of index build completed. 17.7911% of index build completed. 18.0382% of index build completed. 18.2853% of index build completed. 18.5324% of index build completed. 18.7795% of index build completed. 19.0266% of index build completed. 19.2737% of index build completed. 19.5208% of index build completed. 19.7679% of index build completed. 20.015% of index build completed. 20.2621% of index build completed. 20.5092% of index build completed. 20.7563% of index build completed. 21.0034% of index build completed. 21.2505% of index build completed. 21.4976% of index build completed. 21.7447% of index build completed. 21.9918% of index build completed. 22.2389% of index build completed. 22.486% of index build completed. 22.7331% of index build completed. 22.9802% of index build completed. 23.2273% of index build completed. 23.4744% of index build completed. 23.7215% of index build completed. 23.9686% of index build completed. 24.2157% of index build completed. 24.4628% of index build completed. 24.7099% of index build completed. 24.957% of index build completed. 25.2041% of index build completed. 25.4512% of index build completed. 25.6983% of index build completed. 25.9454% of index build completed. 26.1925% of index build completed. 26.4396% of index build completed. 26.6867% of index build completed. 26.9338% of index build completed. 27.1809% of index build completed. 27.428% of index build completed. 27.6751% of index build completed. 27.9222% of index build completed. 28.1693% of index build completed. 28.4164% of index build completed. 28.6635% of index build completed. 28.9106% of index build completed. 29.1577% of index build completed. 29.4048% of index build completed. 29.6519% of index build completed. 29.899% of index build completed. 30.1461% of index build completed. 30.3932% of index build completed. 30.6403% of index build completed. 30.8874% of index build completed. 31.1345% of index build completed. 31.3816% of index build completed. 31.6287% of index build completed. 31.8758% of index build completed. 32.1229% of index build completed. 32.37% of index build completed. 32.6171% of index build completed. 32.8642% of index build completed. 33.1113% of index build completed. 33.3584% of index build completed. 33.6055% of index build completed. 33.8526% of index build completed. 34.0997% of index build completed. 34.3468% of index build completed. 34.5939% of index build completed. 34.841% of index build completed. 35.0881% of index build completed. 35.3352% of index build completed. 35.5823% of index build completed. 35.8294% of index build completed. 36.0765% of index build completed. 36.3236% of index build completed. 36.5707% of index build completed. 36.8178% of index build completed. 37.0649% of index build completed. 37.3119% of index build completed. 37.559% of index build completed. 37.8061% of index build completed. 38.0532% of index build completed. 38.3003% of index build completed. 38.5474% of index build completed. 38.7945% of index build completed. 39.0416% of index build completed. 39.2887% of index build completed. 39.5358% of index build completed. 39.7829% of index build completed. 40.03% of index build completed. 40.2771% of index build completed. 40.5242% of index build completed. 40.7713% of index build completed. 41.0184% of index build completed. 41.2655% of index build completed. 41.5126% of index build completed. 41.7597% of index build completed. 42.0068% of index build completed. 42.2539% of index build completed. 42.501% of index build completed. 42.7481% of index build completed. 42.9952% of index build completed. 43.2423% of index build completed. 43.4894% of index build completed. 43.7365% of index build completed. 43.9836% of index build completed. 44.2307% of index build completed. 44.4778% of index build completed. 44.7249% of index build completed. 44.972% of index build completed. 45.2191% of index build completed. 45.4662% of index build completed. 45.7133% of index build completed. 45.9604% of index build completed. 46.2075% of index build completed. 46.4546% of index build completed. 46.7017% of index build completed. 46.9488% of index build completed. 47.1959% of index build completed. 47.443% of index build completed. 47.6901% of index build completed. 47.9372% of index build completed. 48.1843% of index build completed. 48.4314% of index build completed. 48.6785% of index build completed. 48.9256% of index build completed. 49.1727% of index build completed. 49.4198% of index build completed. 49.6669% of index build completed. 49.914% of index build completed. 50.1611% of index build completed. 50.4082% of index build completed. 50.6553% of index build completed. 50.9024% of index build completed. 51.1495% of index build completed. 51.3966% of index build completed. 51.6437% of index build completed. 51.8908% of index build completed. 52.1379% of index build completed. 52.385% of index build completed. 52.6321% of index build completed. 52.8792% of index build completed. 53.1263% of index build completed. 53.3734% of index build completed. 53.6205% of index build completed. 53.8676% of index build completed. 54.1147% of index build completed. 54.3618% of index build completed. 54.6089% of index build completed. 54.856% of index build completed. 55.1031% of index build completed. 55.3502% of index build completed. 55.5973% of index build completed. 55.8444% of index build completed. 56.0915% of index build completed. 56.3386% of index build completed. 56.5857% of index build completed. 56.8328% of index build completed. 57.0799% of index build completed. 57.327% of index build completed. 57.5741% of index build completed. 57.8212% of index build completed. 58.0683% of index build completed. 58.3154% of index build completed. 58.5625% of index build completed. 58.8096% of index build completed. 59.0567% of index build completed. 59.3038% of index build completed. 59.5509% of index build completed. 59.798% of index build completed. 60.0451% of index build completed. 60.2922% of index build completed. 60.5393% of index build completed. 60.7864% of index build completed. 61.0335% of index build completed. 61.2806% of index build completed. 61.5277% of index build completed. 61.7748% of index build completed. 62.0218% of index build completed. 62.2689% of index build completed. 62.516% of index build completed. 62.7631% of index build completed. 63.0102% of index build completed. 63.2573% of index build completed. 63.5044% of index build completed. 63.7515% of index build completed. 63.9986% of index build completed. 64.2457% of index build completed. 64.4928% of index build completed. 64.7399% of index build completed. 64.987% of index build completed. 65.2341% of index build completed. 65.4812% of index build completed. 65.7283% of index build completed. 65.9754% of index build completed. 66.2225% of index build completed. 66.4696% of index build completed. 66.7167% of index build completed. 66.9638% of index build completed. 67.2109% of index build completed. 67.458% of index build completed. 67.7051% of index build completed. 67.9522% of index build completed. 68.1993% of index build completed. 68.4464% of index build completed. 68.6935% of index build completed. 68.9406% of index build completed. 69.1877% of index build completed. 69.4348% of index build completed. 69.6819% of index build completed. 69.929% of index build completed. 70.1761% of index build completed. 70.4232% of index build completed. 70.6703% of index build completed. 70.9174% of index build completed. 71.1645% of index build completed. 71.4116% of index build completed. 71.6587% of index build completed. 71.9058% of index build completed. 72.1529% of index build completed. 72.4% of index build completed. 72.6471% of index build completed. 72.8942% of index build completed. 73.1413% of index build completed. 73.3884% of index build completed. 73.6355% of index build completed. 73.8826% of index build completed. 74.1297% of index build completed. 74.3768% of index build completed. 74.6239% of index build completed. 74.871% of index build completed. 75.1181% of index build completed. 75.3652% of index build completed. 75.6123% of index build completed. 75.8594% of index build completed. 76.1065% of index build completed. 76.3536% of index build completed. 76.6007% of index build completed. 76.8478% of index build completed. 77.0949% of index build completed. 77.342% of index build completed. 77.5891% of index build completed. 77.8362% of index build completed. 78.0833% of index build completed. 78.3304% of index build completed. 78.5775% of index build completed. 78.8246% of index build completed. 79.0717% of index build completed. 79.3188% of index build completed. 79.5659% of index build completed. 79.813% of index build completed. 80.0601% of index build completed. 80.3072% of index build completed. 80.5543% of index build completed. 80.8014% of index build completed. 81.0485% of index build completed. 81.2956% of index build completed. 81.5427% of index build completed. 81.7898% of index build completed. 82.0369% of index build completed. 82.284% of index build completed. 82.5311% of index build completed. 82.7782% of index build completed. 83.0253% of index build completed. 83.2724% of index build completed. 83.5195% of index build completed. 83.7666% of index build completed. 84.0137% of index build completed. 84.2608% of index build completed. 84.5079% of index build completed. 84.755% of index build completed. 85.0021% of index build completed. 85.2492% of index build completed. 85.4963% of index build completed. 85.7434% of index build completed. 85.9905% of index build completed. 86.2376% of index build completed. 86.4847% of index build completed. 86.7317% of index build completed. 86.9788% of index build completed. 87.2259% of index build completed. 87.473% of index build completed. 87.7201% of index build completed. 87.9672% of index build completed. 88.2143% of index build completed. 88.4614% of index build completed. 88.7085% of index build completed. 88.9556% of index build completed. 89.2027% of index build completed. 89.4498% of index build completed. 89.6969% of index build completed. 89.944% of index build completed. 90.1911% of index build completed. 90.4382% of index build completed. 90.6853% of index build completed. 90.9324% of index build completed. 91.1795% of index build completed. 91.4266% of index build completed. 91.6737% of index build completed. 91.9208% of index build completed. 92.1679% of index build completed. 92.415% of index build completed. 92.6621% of index build completed. 92.9092% of index build completed. 93.1563% of index build completed. 93.4034% of index build completed. 93.6505% of index build completed. 93.8976% of index build completed. 94.1447% of index build completed. 94.3918% of index build completed. 94.6389% of index build completed. 94.886% of index build completed. 95.1331% of index build completed. 95.3802% of index build completed. 95.6273% of index build completed. 95.8744% of index build completed. 96.1215% of index build completed. 96.3686% of index build completed. 96.6157% of index build completed. 96.8628% of index build completed. 97.1099% of index build completed. 97.357% of index build completed. 97.6041% of index build completed. 97.8512% of index build completed. 98.0983% of index build completed. 98.3454% of index build completed. 98.5925% of index build completed. 98.8396% of index build completed. 99.0867% of index build completed. 99.3338% of index build completed. 99.5809% of index build completed. 99.828% of index build completed.Starting final cleanup..done. Link time: 3495.63s +Index built with degree: max:64 avg:44.9527 min:1 count(deg<2):1019259 +Robust Prune Calls: 108594089 +Indexing time: 3504.72 +Not saving tags as they are not enabled. +Time taken for save: 19.6696s. diff --git a/plot.py b/plot.py new file mode 100644 index 000000000..111ecfb49 --- /dev/null +++ b/plot.py @@ -0,0 +1,85 @@ +import matplotlib.pyplot as plt + +# Data for different configurations +configurations = [ + { + 'label': 'baseline', + 'recall': [80.60, 88.66, 92.12, 94.05, 95.26, 96.09], + 'QPS': [204.00, 122.17, 90.58, 73.30, 62.50, 55.09], + 'avg_dist_cmps': [7716.33, 13663.56, 19477.48, 25174.82, 30768.75, 36281.84], + 'avg_hops': [59.49, 108.56, 158.08, 207.77, 257.52, 307.35], + 'mean_latency': [4901.96, 8185.00, 11039.73, 13642.69, 15998.88, 18151.68] + }, + { + 'label': 'x=5', + 'recall': [75.56, 82.88, 86.48, 88.72, 90.28, 91.43], + 'QPS': [138.48, 84.11, 68.15, 58.07, 49.56, 44.64], + 'avg_dist_cmps': [5826.41, 8608.48, 10925.24, 13030.90, 14996.30, 16868.74], + 'avg_hops': [44.82, 66.84, 85.45, 102.54, 118.66, 134.20], + 'mean_latency': [7221.22, 11889.23, 14674.08, 17219.57, 20178.49, 22402.45] + }, + { + 'label': 'x=7', + 'recall': [78.15, 85.50, 88.87, 90.86, 92.18, 93.15], + 'QPS': [219.83, 138.20, 105.45, 86.58, 74.60, 66.05], + 'avg_dist_cmps': [6571.48, 10162.82, 13149.50, 15806.06, 18226.08, 20523.22], + 'avg_hops': [50.53, 79.23, 103.48, 125.31, 145.43, 164.73], + 'mean_latency': [4549.00, 7236.04, 9482.81, 11550.14, 13404.36, 15140.81] + }, + { + 'label': 'x=10', + 'recall': [79.69, 87.33, 90.61, 92.49, 93.71, 94.55], + 'QPS': [198.75, 118.69, 88.00, 71.21, 60.84, 53.51], + 'avg_dist_cmps': [7149.92, 11640.27, 15499.88, 18942.36, 22119.38, 25015.15], + 'avg_hops': [55.00, 91.23, 122.88, 151.51, 178.28, 202.91], + 'mean_latency': [5031.34, 8425.49, 11363.82, 14041.94, 16437.29, 18686.43] + } + # Add more configurations here +] + +# Colors for different configurations +colors = ['b', 'g', 'r', 'y'] + +# Plot latency vs recall +plt.figure() +for i, config in enumerate(configurations): + plt.plot(config['mean_latency'], config['recall'], marker='o', color=colors[i % len(colors)], label=config['label']) +plt.xlabel('Mean Latency (mus)') +plt.ylabel('Recall@50') +plt.title('Mean Latency vs Recall') +plt.legend() +plt.grid(True) +plt.savefig('prec40M/latency_vs_recall.png') + +# Plot QPS vs recall +plt.figure() +for i, config in enumerate(configurations): + plt.plot(config['QPS'], config['recall'], marker='o', color=colors[i % len(colors)], label=config['label']) +plt.xlabel('QPS') +plt.ylabel('Recall@50') +plt.title('QPS vs Recall') +plt.legend() +plt.grid(True) +plt.savefig('prec40M/qps_vs_recall.png') + +# Plot hops vs recall +plt.figure() +for i, config in enumerate(configurations): + plt.plot(config['avg_hops'], config['recall'], marker='o', color=colors[i % len(colors)], label=config['label']) +plt.xlabel('Avg Hops') +plt.ylabel('Recall@50') +plt.title('Avg Hops vs Recall') +plt.legend() +plt.grid(True) +plt.savefig('prec40M/hops_vs_recall.png') + +# Plot dist comps vs recall +plt.figure() +for i, config in enumerate(configurations): + plt.plot(config['avg_dist_cmps'], config['recall'], marker='o', color=colors[i % len(colors)], label=config['label']) +plt.xlabel('Avg Dist Comps') +plt.ylabel('Recall@50') +plt.title('Avg Dist Comps vs Recall') +plt.legend() +plt.grid(True) +plt.savefig('prec40M/dist_comps_vs_recall.png') diff --git a/prec40M/build_&_search/early_stop_build_&_search_r150_l300_k50_x10_y0_L.txt b/prec40M/build_&_search/early_stop_build_&_search_r150_l300_k50_x10_y0_L.txt new file mode 100644 index 000000000..db499a82f --- /dev/null +++ b/prec40M/build_&_search/early_stop_build_&_search_r150_l300_k50_x10_y0_L.txt @@ -0,0 +1,84 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.41201s +From graph header, expected_file_size: 17246453220, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_earlystop_memory_index_r150_l300_x_10_y_0_L.......done. Index has 40469609 nodes and 4271143690 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@50 +=================================================================================================== + 50 205.79 6879.70 55.44 4859.33 22760.29 78.55 + 100 124.62 11202.62 92.05 8024.39 27475.83 86.26 + 150 92.43 14903.85 123.93 10818.76 32318.81 89.65 + 200 75.20 18226.06 152.88 13298.50 36122.13 91.56 + 250 64.19 21264.22 179.71 15579.23 40343.23 92.85 + 300 56.38 24139.46 205.31 17736.63 44439.92 93.76 + 350 50.56 26824.08 229.53 19777.54 50537.66 94.43 + 400 46.00 29438.52 253.23 21740.69 51937.45 94.96 + 450 42.21 31964.18 276.29 23689.72 56824.03 95.43 + 500 39.25 34315.37 297.92 25478.36 61234.52 95.79 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_150_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_150_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_200_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_200_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_250_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_250_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_350_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_350_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_400_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_400_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_450_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_450_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_500_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_500_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. diff --git a/prec40M/build_&_search/early_stop_build_r150_l300_x10_y0_L.txt b/prec40M/build_&_search/early_stop_build_r150_l300_x10_y0_L.txt new file mode 100644 index 000000000..2480933cc --- /dev/null +++ b/prec40M/build_&_search/early_stop_build_r150_l300_x10_y0_L.txt @@ -0,0 +1,9 @@ +Starting index build with R: 150 Lbuild: 300 alpha: 1.2 #threads: 48 +Using AVX2 distance computation DistanceL2Int8. +Using only first 40469609 from file.. +Starting index build with 40469609 points... + 0% of index build completed. 0.247099% of index build completed. 0.494198% of index build completed. 0.741297% of index build completed. 0.988396% of index build completed. 1.2355% of index build completed. 1.48259% of index build completed. 1.72969% of index build completed. 1.97679% of index build completed. 2.22389% of index build completed. 2.47099% of index build completed. 2.71809% of index build completed. 2.96519% of index build completed. 3.21229% of index build completed. 3.45939% of index build completed. 3.70649% of index build completed. 3.95358% of index build completed. 4.20068% of index build completed. 4.44778% of index build completed. 4.69488% of index build completed. 4.94198% of index build completed. 5.18908% of index build completed. 5.43618% of index build completed. 5.68328% of index build completed. 5.93038% of index build completed. 6.17748% of index build completed. 6.42457% of index build completed. 6.67167% of index build completed. 6.91877% of index build completed. 7.16587% of index build completed. 7.41297% of index build completed. 7.66007% of index build completed. 7.90717% of index build completed. 8.15427% of index build completed. 8.40137% of index build completed. 8.64847% of index build completed. 8.89556% of index build completed. 9.14266% of index build completed. 9.38976% of index build completed. 9.63686% of index build completed. 9.88396% of index build completed. 10.1311% of index build completed. 10.3782% of index build completed. 10.6253% of index build completed. 10.8724% of index build completed. 11.1195% of index build completed. 11.3666% of index build completed. 11.6137% of index build completed. 11.8608% of index build completed. 12.1079% of index build completed. 12.355% of index build completed. 12.602% of index build completed. 12.8491% of index build completed. 13.0962% of index build completed. 13.3433% of index build completed. 13.5904% of index build completed. 13.8375% of index build completed. 14.0846% of index build completed. 14.3317% of index build completed. 14.5788% of index build completed. 14.8259% of index build completed. 15.073% of index build completed. 15.3201% of index build completed. 15.5672% of index build completed. 15.8143% of index build completed. 16.0614% of index build completed. 16.3085% of index build completed. 16.5556% of index build completed. 16.8027% of index build completed. 17.0498% of index build completed. 17.2969% of index build completed. 17.544% of index build completed. 17.7911% of index build completed. 18.0382% of index build completed. 18.2853% of index build completed. 18.5324% of index build completed. 18.7795% of index build completed. 19.0266% of index build completed. 19.2737% of index build completed. 19.5208% of index build completed. 19.7679% of index build completed. 20.015% of index build completed. 20.2621% of index build completed. 20.5092% of index build completed. 20.7563% of index build completed. 21.0034% of index build completed. 21.2505% of index build completed. 21.4976% of index build completed. 21.7447% of index build completed. 21.9918% of index build completed. 22.2389% of index build completed. 22.486% of index build completed. 22.7331% of index build completed. 22.9802% of index build completed. 23.2273% of index build completed. 23.4744% of index build completed. 23.7215% of index build completed. 23.9686% of index build completed. 24.2157% of index build completed. 24.4628% of index build completed. 24.7099% of index build completed. 24.957% of index build completed. 25.2041% of index build completed. 25.4512% of index build completed. 25.6983% of index build completed. 25.9454% of index build completed. 26.1925% of index build completed. 26.4396% of index build completed. 26.6867% of index build completed. 26.9338% of index build completed. 27.1809% of index build completed. 27.428% of index build completed. 27.6751% of index build completed. 27.9222% of index build completed. 28.1693% of index build completed. 28.4164% of index build completed. 28.6635% of index build completed. 28.9106% of index build completed. 29.1577% of index build completed. 29.4048% of index build completed. 29.6519% of index build completed. 29.899% of index build completed. 30.1461% of index build completed. 30.3932% of index build completed. 30.6403% of index build completed. 30.8874% of index build completed. 31.1345% of index build completed. 31.3816% of index build completed. 31.6287% of index build completed. 31.8758% of index build completed. 32.1229% of index build completed. 32.37% of index build completed. 32.6171% of index build completed. 32.8642% of index build completed. 33.1113% of index build completed. 33.3584% of index build completed. 33.6055% of index build completed. 33.8526% of index build completed. 34.0997% of index build completed. 34.3468% of index build completed. 34.5939% of index build completed. 34.841% of index build completed. 35.0881% of index build completed. 35.3352% of index build completed. 35.5823% of index build completed. 35.8294% of index build completed. 36.0765% of index build completed. 36.3236% of index build completed. 36.5707% of index build completed. 36.8178% of index build completed. 37.0649% of index build completed. 37.3119% of index build completed. 37.559% of index build completed. 37.8061% of index build completed. 38.0532% of index build completed. 38.3003% of index build completed. 38.5474% of index build completed. 38.7945% of index build completed. 39.0416% of index build completed. 39.2887% of index build completed. 39.5358% of index build completed. 39.7829% of index build completed. 40.03% of index build completed. 40.2771% of index build completed. 40.5242% of index build completed. 40.7713% of index build completed. 41.0184% of index build completed. 41.2655% of index build completed. 41.5126% of index build completed. 41.7597% of index build completed. 42.0068% of index build completed. 42.2539% of index build completed. 42.501% of index build completed. 42.7481% of index build completed. 42.9952% of index build completed. 43.2423% of index build completed. 43.4894% of index build completed. 43.7365% of index build completed. 43.9836% of index build completed. 44.2307% of index build completed. 44.4778% of index build completed. 44.7249% of index build completed. 44.972% of index build completed. 45.2191% of index build completed. 45.4662% of index build completed. 45.7133% of index build completed. 45.9604% of index build completed. 46.2075% of index build completed. 46.4546% of index build completed. 46.7017% of index build completed. 46.9488% of index build completed. 47.1959% of index build completed. 47.443% of index build completed. 47.6901% of index build completed. 47.9372% of index build completed. 48.1843% of index build completed. 48.4314% of index build completed. 48.6785% of index build completed. 48.9256% of index build completed. 49.1727% of index build completed. 49.4198% of index build completed. 49.6669% of index build completed. 49.914% of index build completed. 50.1611% of index build completed. 50.4082% of index build completed. 50.6553% of index build completed. 50.9024% of index build completed. 51.1495% of index build completed. 51.3966% of index build completed. 51.6437% of index build completed. 51.8908% of index build completed. 52.1379% of index build completed. 52.385% of index build completed. 52.6321% of index build completed. 52.8792% of index build completed. 53.1263% of index build completed. 53.3734% of index build completed. 53.6205% of index build completed. 53.8676% of index build completed. 54.1147% of index build completed. 54.3618% of index build completed. 54.6089% of index build completed. 54.856% of index build completed. 55.1031% of index build completed. 55.3502% of index build completed. 55.5973% of index build completed. 55.8444% of index build completed. 56.0915% of index build completed. 56.3386% of index build completed. 56.5857% of index build completed. 56.8328% of index build completed. 57.0799% of index build completed. 57.327% of index build completed. 57.5741% of index build completed. 57.8212% of index build completed. 58.0683% of index build completed. 58.3154% of index build completed. 58.5625% of index build completed. 58.8096% of index build completed. 59.0567% of index build completed. 59.3038% of index build completed. 59.5509% of index build completed. 59.798% of index build completed. 60.0451% of index build completed. 60.2922% of index build completed. 60.5393% of index build completed. 60.7864% of index build completed. 61.0335% of index build completed. 61.2806% of index build completed. 61.5277% of index build completed. 61.7748% of index build completed. 62.0218% of index build completed. 62.2689% of index build completed. 62.516% of index build completed. 62.7631% of index build completed. 63.0102% of index build completed. 63.2573% of index build completed. 63.5044% of index build completed. 63.7515% of index build completed. 63.9986% of index build completed. 64.2457% of index build completed. 64.4928% of index build completed. 64.7399% of index build completed. 64.987% of index build completed. 65.2341% of index build completed. 65.4812% of index build completed. 65.7283% of index build completed. 65.9754% of index build completed. 66.2225% of index build completed. 66.4696% of index build completed. 66.7167% of index build completed. 66.9638% of index build completed. 67.2109% of index build completed. 67.458% of index build completed. 67.7051% of index build completed. 67.9522% of index build completed. 68.1993% of index build completed. 68.4464% of index build completed. 68.6935% of index build completed. 68.9406% of index build completed. 69.1877% of index build completed. 69.4348% of index build completed. 69.6819% of index build completed. 69.929% of index build completed. 70.1761% of index build completed. 70.4232% of index build completed. 70.6703% of index build completed. 70.9174% of index build completed. 71.1645% of index build completed. 71.4116% of index build completed. 71.6587% of index build completed. 71.9058% of index build completed. 72.1529% of index build completed. 72.4% of index build completed. 72.6471% of index build completed. 72.8942% of index build completed. 73.1413% of index build completed. 73.3884% of index build completed. 73.6355% of index build completed. 73.8826% of index build completed. 74.1297% of index build completed. 74.3768% of index build completed. 74.6239% of index build completed. 74.871% of index build completed. 75.1181% of index build completed. 75.3652% of index build completed. 75.6123% of index build completed. 75.8594% of index build completed. 76.1065% of index build completed. 76.3536% of index build completed. 76.6007% of index build completed. 76.8478% of index build completed. 77.0949% of index build completed. 77.342% of index build completed. 77.5891% of index build completed. 77.8362% of index build completed. 78.0833% of index build completed. 78.3304% of index build completed. 78.5775% of index build completed. 78.8246% of index build completed. 79.0717% of index build completed. 79.3188% of index build completed. 79.5659% of index build completed. 79.813% of index build completed. 80.0601% of index build completed. 80.3072% of index build completed. 80.5543% of index build completed. 80.8014% of index build completed. 81.0485% of index build completed. 81.2956% of index build completed. 81.5427% of index build completed. 81.7898% of index build completed. 82.0369% of index build completed. 82.284% of index build completed. 82.5311% of index build completed. 82.7782% of index build completed. 83.0253% of index build completed. 83.2724% of index build completed. 83.5195% of index build completed. 83.7666% of index build completed. 84.0137% of index build completed. 84.2608% of index build completed. 84.5079% of index build completed. 84.755% of index build completed. 85.0021% of index build completed. 85.2492% of index build completed. 85.4963% of index build completed. 85.7434% of index build completed. 85.9905% of index build completed. 86.2376% of index build completed. 86.4847% of index build completed. 86.7317% of index build completed. 86.9788% of index build completed. 87.2259% of index build completed. 87.473% of index build completed. 87.7201% of index build completed. 87.9672% of index build completed. 88.2143% of index build completed. 88.4614% of index build completed. 88.7085% of index build completed. 88.9556% of index build completed. 89.2027% of index build completed. 89.4498% of index build completed. 89.6969% of index build completed. 89.944% of index build completed. 90.1911% of index build completed. 90.4382% of index build completed. 90.6853% of index build completed. 90.9324% of index build completed. 91.1795% of index build completed. 91.4266% of index build completed. 91.6737% of index build completed. 91.9208% of index build completed. 92.1679% of index build completed. 92.415% of index build completed. 92.6621% of index build completed. 92.9092% of index build completed. 93.1563% of index build completed. 93.4034% of index build completed. 93.6505% of index build completed. 93.8976% of index build completed. 94.1447% of index build completed. 94.3918% of index build completed. 94.6389% of index build completed. 94.886% of index build completed. 95.1331% of index build completed. 95.3802% of index build completed. 95.6273% of index build completed. 95.8744% of index build completed. 96.1215% of index build completed. 96.3686% of index build completed. 96.6157% of index build completed. 96.8628% of index build completed. 97.1099% of index build completed. 97.357% of index build completed. 97.6041% of index build completed. 97.8512% of index build completed. 98.0983% of index build completed. 98.3454% of index build completed. 98.5925% of index build completed. 98.8396% of index build completed. 99.0867% of index build completed. 99.3338% of index build completed. 99.5809% of index build completed. 99.828% of index build completed.Starting final cleanup..done. Link time: 14110.7s +Index built with degree: max:150 avg:105.54 min:1 count(deg<2):574870 +Indexing time: 14120.8 +Not saving tags as they are not enabled. +Time taken for save: 28.7461s. diff --git a/prec40M/dist_comps_vs_recall.png b/prec40M/dist_comps_vs_recall.png new file mode 100644 index 000000000..3198dc116 Binary files /dev/null and b/prec40M/dist_comps_vs_recall.png differ diff --git a/prec40M/distance-based-stop/early_stop_1pc_r64_k10.txt b/prec40M/distance-based-stop/early_stop_1pc_r64_k10.txt new file mode 100644 index 000000000..746101b85 --- /dev/null +++ b/prec40M/distance-based-stop/early_stop_1pc_r64_k10.txt @@ -0,0 +1,84 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.38647s +From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@10 +=================================================================================================== + 10 5601.33 331.51 7.73 178.50 1213.03 6.83 + 20 7570.89 332.91 7.77 132.06 872.51 6.86 + 30 7235.70 332.92 7.77 138.18 850.40 6.86 + 40 7103.38 332.92 7.77 140.75 857.32 6.86 + 50 6943.05 332.92 7.77 144.00 868.61 6.86 + 60 6923.15 332.92 7.77 144.41 870.57 6.86 + 70 6128.61 357.67 8.26 163.14 938.15 7.44 + 80 5765.07 381.73 8.75 173.43 936.66 8.22 + 90 5597.03 392.64 8.97 178.64 939.07 8.60 + 100 4863.46 438.01 10.00 205.59 956.61 9.65 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_10_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_10_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_20_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_20_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_30_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_30_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_40_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_40_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. diff --git a/prec40M/distance-based-stop/early_stop_1pc_r64_k100.txt b/prec40M/distance-based-stop/early_stop_1pc_r64_k100.txt new file mode 100644 index 000000000..3b556b8a4 --- /dev/null +++ b/prec40M/distance-based-stop/early_stop_1pc_r64_k100.txt @@ -0,0 +1,21 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.42052s +From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@100 +=================================================================================================== + 100 3806.55 438.01 10.00 262.67 1432.43 5.57 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 100, size = 5306008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 100, size = 5306008B +Finished writing bin. diff --git a/prec40M/distance-based-stop/early_stop_1pc_r64_k25.txt b/prec40M/distance-based-stop/early_stop_1pc_r64_k25.txt new file mode 100644 index 000000000..729e13f5e --- /dev/null +++ b/prec40M/distance-based-stop/early_stop_1pc_r64_k25.txt @@ -0,0 +1,70 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.41602s +From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@25 +=================================================================================================== + 30 5281.62 332.92 7.77 189.31 1354.39 5.83 + 40 7050.04 332.92 7.77 141.82 863.16 5.83 + 50 6997.94 332.92 7.77 142.87 877.10 5.83 + 60 6873.09 332.92 7.77 145.47 868.20 5.83 + 70 6157.34 357.67 8.26 162.38 951.35 6.32 + 80 5790.40 381.73 8.75 172.67 927.46 7.00 + 90 5606.04 392.64 8.97 178.35 900.73 7.33 + 100 4926.95 438.01 10.00 202.94 953.73 8.21 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_30_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_30_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_40_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_40_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. diff --git a/prec40M/distance-based-stop/early_stop_1pc_r64_k50.txt b/prec40M/distance-based-stop/early_stop_1pc_r64_k50.txt new file mode 100644 index 000000000..9e6475cb2 --- /dev/null +++ b/prec40M/distance-based-stop/early_stop_1pc_r64_k50.txt @@ -0,0 +1,56 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.41114s +From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@50 +=================================================================================================== + 50 5148.20 332.92 7.77 194.21 1346.07 4.94 + 60 6898.92 332.92 7.77 144.92 860.76 4.94 + 70 6218.21 357.67 8.26 160.79 930.09 5.34 + 80 5745.99 381.73 8.75 174.01 910.59 5.91 + 90 5635.25 392.64 8.97 177.43 923.18 6.18 + 100 4940.16 438.01 10.00 202.39 984.91 6.93 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. diff --git a/prec40M/hops_vs_recall.png b/prec40M/hops_vs_recall.png new file mode 100644 index 000000000..5889f48d0 Binary files /dev/null and b/prec40M/hops_vs_recall.png differ diff --git a/prec40M/latency_vs_recall.png b/prec40M/latency_vs_recall.png new file mode 100644 index 000000000..28d383740 Binary files /dev/null and b/prec40M/latency_vs_recall.png differ diff --git a/prec40M/qps_vs_recall.png b/prec40M/qps_vs_recall.png new file mode 100644 index 000000000..350ce899e Binary files /dev/null and b/prec40M/qps_vs_recall.png differ diff --git a/prec40M/search_only/early_stop_search_r150_l300_k50_x10_y0_K.txt b/prec40M/search_only/early_stop_search_r150_l300_k50_x10_y0_K.txt new file mode 100644 index 000000000..7af630670 --- /dev/null +++ b/prec40M/search_only/early_stop_search_r150_l300_k50_x10_y0_K.txt @@ -0,0 +1,84 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 2.54451s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@50 +=================================================================================================== + 50 103.14 7149.92 55.00 9695.56 43790.95 79.69 + 100 93.59 8718.79 67.67 10685.09 43798.63 83.13 + 150 83.29 8828.62 68.56 12006.17 63147.71 83.29 + 200 69.66 8831.40 68.59 14354.46 98533.82 83.30 + 250 79.34 8831.58 68.59 12603.72 60889.81 83.30 + 300 86.38 8831.58 68.59 11576.72 45677.29 83.30 + 350 78.09 8831.58 68.59 12805.55 67069.02 83.30 + 400 77.27 8831.58 68.59 12941.84 63518.28 83.30 + 450 76.49 8831.58 68.59 13073.44 64098.41 83.30 + 500 73.66 8831.58 68.59 13575.05 73795.97 83.30 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_150_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_150_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_200_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_200_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_250_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_250_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_350_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_350_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_400_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_400_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_450_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_450_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_500_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_500_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_search_r150_l300_k50_x10_y0_L.txt b/prec40M/search_only/early_stop_search_r150_l300_k50_x10_y0_L.txt new file mode 100644 index 000000000..3d3e21465 --- /dev/null +++ b/prec40M/search_only/early_stop_search_r150_l300_k50_x10_y0_L.txt @@ -0,0 +1,84 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.35261s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@50 +=================================================================================================== + 50 198.75 7149.92 55.00 5031.34 22593.96 79.69 + 100 118.69 11640.27 91.23 8425.49 27716.19 87.33 + 150 88.00 15499.88 122.88 11363.82 32365.00 90.61 + 200 71.21 18942.36 151.51 14041.94 36769.11 92.49 + 250 60.84 22119.38 178.28 16437.29 41746.58 93.71 + 300 53.51 25015.15 202.91 18686.43 46725.85 94.55 + 350 47.99 27811.66 226.88 20837.03 51399.47 95.19 + 400 43.52 30522.28 250.33 22975.24 56028.80 95.70 + 450 40.11 33087.46 272.69 24931.99 60298.46 96.11 + 500 37.61 35495.27 293.78 26588.03 63596.32 96.43 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_150_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_150_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_200_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_200_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_250_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_250_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_350_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_350_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_400_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_400_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_450_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_450_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_500_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_500_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_search_r150_l300_k50_x5_y0_K.txt b/prec40M/search_only/early_stop_search_r150_l300_k50_x5_y0_K.txt new file mode 100644 index 000000000..cb3a2e15a --- /dev/null +++ b/prec40M/search_only/early_stop_search_r150_l300_k50_x5_y0_K.txt @@ -0,0 +1,84 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 2.67896s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@50 +=================================================================================================== + 50 92.70 5826.41 44.82 10787.55 54707.85 75.56 + 100 103.06 5992.70 46.22 9702.70 50245.97 76.03 + 150 118.29 5992.72 46.23 8453.61 43668.85 76.03 + 200 83.78 5992.72 46.23 11935.34 102525.97 76.03 + 250 114.22 5992.72 46.23 8754.73 45815.66 76.03 + 300 90.91 5992.72 46.23 10999.88 59534.30 76.03 + 350 100.15 5992.72 46.23 9985.23 49787.56 76.03 + 400 110.68 5992.72 46.23 9034.44 45854.38 76.03 + 450 97.91 5992.72 46.23 10213.48 51556.87 76.03 + 500 101.96 5992.72 46.23 9807.39 48884.99 76.03 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_150_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_150_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_200_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_200_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_250_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_250_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_350_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_350_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_400_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_400_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_450_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_450_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_500_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_500_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_search_r150_l300_k50_x5_y0_L.txt b/prec40M/search_only/early_stop_search_r150_l300_k50_x5_y0_L.txt new file mode 100644 index 000000000..305dc9af8 --- /dev/null +++ b/prec40M/search_only/early_stop_search_r150_l300_k50_x5_y0_L.txt @@ -0,0 +1,56 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 2.72896s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@50 +=================================================================================================== + 50 143.87 5826.41 44.82 6950.25 32920.13 75.56 + 100 85.51 8608.48 66.84 11693.88 54980.88 82.88 + 150 74.34 10925.24 85.45 13450.85 44510.98 86.48 + 200 58.23 13030.90 102.54 17171.86 88702.41 88.72 + 250 46.48 14996.30 118.66 21516.11 118291.45 90.28 + 300 42.52 16868.74 134.20 23517.47 139291.89 91.43 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_150_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_150_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_200_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_200_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_250_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_250_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_search_r150_l300_k50_x7_y0_L.txt b/prec40M/search_only/early_stop_search_r150_l300_k50_x7_y0_L.txt new file mode 100644 index 000000000..201faa774 --- /dev/null +++ b/prec40M/search_only/early_stop_search_r150_l300_k50_x7_y0_L.txt @@ -0,0 +1,56 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.39078s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@50 +=================================================================================================== + 50 219.83 6571.48 50.53 4549.00 22603.01 78.15 + 100 138.20 10162.82 79.23 7236.04 27292.48 85.50 + 150 105.45 13149.50 103.48 9482.81 29771.95 88.87 + 200 86.58 15806.06 125.31 11550.14 34007.52 90.86 + 250 74.60 18226.08 145.43 13404.36 36635.31 92.18 + 300 66.05 20523.22 164.73 15140.81 41088.31 93.15 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_150_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_150_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_200_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_200_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_250_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_250_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_x10_y0_K/early_stop_r150_k10.txt b/prec40M/search_only/early_stop_x10_y0_K/early_stop_r150_k10.txt new file mode 100644 index 000000000..2e631b6e4 --- /dev/null +++ b/prec40M/search_only/early_stop_x10_y0_K/early_stop_r150_k10.txt @@ -0,0 +1,21 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.34163s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@10 +=================================================================================================== + 300 297.52 4601.87 35.50 3361.06 18919.91 83.78 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_x10_y0_K/early_stop_r150_k100.txt b/prec40M/search_only/early_stop_x10_y0_K/early_stop_r150_k100.txt new file mode 100644 index 000000000..a8b414d72 --- /dev/null +++ b/prec40M/search_only/early_stop_x10_y0_K/early_stop_r150_k100.txt @@ -0,0 +1,21 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.33751s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@100 +=================================================================================================== + 300 102.53 12864.34 101.10 9753.06 31099.98 83.13 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 100, size = 5306008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 100, size = 5306008B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_x10_y0_K/early_stop_r150_k30.txt b/prec40M/search_only/early_stop_x10_y0_K/early_stop_r150_k30.txt new file mode 100644 index 000000000..ad8fbfaad --- /dev/null +++ b/prec40M/search_only/early_stop_x10_y0_K/early_stop_r150_k30.txt @@ -0,0 +1,21 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.39128s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@30 +=================================================================================================== + 300 194.45 6906.99 53.39 5142.73 21151.91 83.44 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 30, size = 1591808B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 30, size = 1591808B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_x10_y0_K/early_stop_r150_k50.txt b/prec40M/search_only/early_stop_x10_y0_K/early_stop_r150_k50.txt new file mode 100644 index 000000000..5b4b8ec16 --- /dev/null +++ b/prec40M/search_only/early_stop_x10_y0_K/early_stop_r150_k50.txt @@ -0,0 +1,21 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.34222s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@50 +=================================================================================================== + 300 151.71 8831.58 68.59 6591.33 25331.64 83.30 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_x10_y0_L/early_stop_r150_k10.txt b/prec40M/search_only/early_stop_x10_y0_L/early_stop_r150_k10.txt new file mode 100644 index 000000000..ebae58ffa --- /dev/null +++ b/prec40M/search_only/early_stop_x10_y0_L/early_stop_r150_k10.txt @@ -0,0 +1,21 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.34877s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@10 +=================================================================================================== + 300 53.56 25015.15 202.91 18671.99 43509.42 97.93 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_x10_y0_L/early_stop_r150_k100.txt b/prec40M/search_only/early_stop_x10_y0_L/early_stop_r150_k100.txt new file mode 100644 index 000000000..0c32db705 --- /dev/null +++ b/prec40M/search_only/early_stop_x10_y0_L/early_stop_r150_k100.txt @@ -0,0 +1,21 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.34179s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@100 +=================================================================================================== + 300 53.51 25015.15 202.91 18689.31 44566.99 91.35 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 100, size = 5306008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 100, size = 5306008B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_x10_y0_L/early_stop_r150_k30.txt b/prec40M/search_only/early_stop_x10_y0_L/early_stop_r150_k30.txt new file mode 100644 index 000000000..0862aa842 --- /dev/null +++ b/prec40M/search_only/early_stop_x10_y0_L/early_stop_r150_k30.txt @@ -0,0 +1,21 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.32514s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@30 +=================================================================================================== + 300 53.30 25015.15 202.91 18761.52 43157.76 96.07 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 30, size = 1591808B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 30, size = 1591808B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_x10_y0_L/early_stop_r150_k50.txt b/prec40M/search_only/early_stop_x10_y0_L/early_stop_r150_k50.txt new file mode 100644 index 000000000..173a381cb --- /dev/null +++ b/prec40M/search_only/early_stop_x10_y0_L/early_stop_r150_k50.txt @@ -0,0 +1,21 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.31941s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@50 +=================================================================================================== + 300 53.39 25015.15 202.91 18730.34 43019.43 94.55 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_x5_y0_K/early_stop_r150_k10.txt b/prec40M/search_only/early_stop_x5_y0_K/early_stop_r150_k10.txt new file mode 100644 index 000000000..ffc3ac47c --- /dev/null +++ b/prec40M/search_only/early_stop_x5_y0_K/early_stop_r150_k10.txt @@ -0,0 +1,21 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.34911s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@10 +=================================================================================================== + 300 419.11 3264.07 25.29 2385.95 16366.58 75.74 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_x5_y0_K/early_stop_r150_k100.txt b/prec40M/search_only/early_stop_x5_y0_K/early_stop_r150_k100.txt new file mode 100644 index 000000000..4d7789c11 --- /dev/null +++ b/prec40M/search_only/early_stop_x5_y0_K/early_stop_r150_k100.txt @@ -0,0 +1,21 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.34336s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@100 +=================================================================================================== + 300 154.61 8652.72 67.20 6468.04 23300.55 76.09 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 100, size = 5306008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 100, size = 5306008B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_x5_y0_K/early_stop_r150_k30.txt b/prec40M/search_only/early_stop_x5_y0_K/early_stop_r150_k30.txt new file mode 100644 index 000000000..9e86cfc72 --- /dev/null +++ b/prec40M/search_only/early_stop_x5_y0_K/early_stop_r150_k30.txt @@ -0,0 +1,21 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.34291s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@30 +=================================================================================================== + 300 281.03 4781.82 36.85 3558.22 19045.44 76.26 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 30, size = 1591808B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 30, size = 1591808B +Finished writing bin. diff --git a/prec40M/search_only/early_stop_x5_y0_K/early_stop_r150_k50.txt b/prec40M/search_only/early_stop_x5_y0_K/early_stop_r150_k50.txt new file mode 100644 index 000000000..ef52d025f --- /dev/null +++ b/prec40M/search_only/early_stop_x5_y0_K/early_stop_r150_k50.txt @@ -0,0 +1,21 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.35146s +From graph header, expected_file_size: 18732710752, _max_observed_degree: 150, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300.......done. Index has 40469609 nodes and 4642708073 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 1 threads to search + Ls QPS Avg dist cmps Avg hops Mean Latency (mus) 99.9 Latency Recall@50 +=================================================================================================== + 300 220.81 5992.72 46.23 4528.76 20761.37 76.03 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_300_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_300_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. diff --git a/run-build.sh b/run-build.sh new file mode 100644 index 000000000..62abd468c --- /dev/null +++ b/run-build.sh @@ -0,0 +1,2 @@ +cd build +./apps/build_memory_index --data_path /nvmessd1/fbv4/prec40M.bin --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_earlystop_memory_index_r150_l300_x_10_y_0_L -R 150 -L 300 -T 48 > /home/rakri/avarhade/DiskANN/prec40M/early-stop-build_r150_l300_x10_y0_L.txt \ No newline at end of file diff --git a/run-search.sh b/run-search.sh new file mode 100644 index 000000000..3869bcaf6 --- /dev/null +++ b/run-search.sh @@ -0,0 +1,6 @@ +cd build +# ./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300 --gt_file /nvmessd1/fbv4/gt100_prec40M --query_file /nvmessd1/fbv4/queries384d.bin --result_path /home/rakri/avarhade/Dump/tmp -K 10 -L 300 -T 1 > /home/rakri/avarhade/DiskANN/prec40M/early-stop-build-x-10-y-0/early_stop_r150_k10.txt +# ./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300 --gt_file /nvmessd1/fbv4/gt100_prec40M --query_file /nvmessd1/fbv4/queries384d.bin --result_path /home/rakri/avarhade/Dump/tmp -K 30 -L 300 -T 1 > /home/rakri/avarhade/DiskANN/prec40M/early-stop-build-x-10-y-0/early_stop_r150_k30.txt +# ./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300 --gt_file /nvmessd1/fbv4/gt100_prec40M --query_file /nvmessd1/fbv4/queries384d.bin --result_path /home/rakri/avarhade/Dump/tmp -K 50 -L 300 -T 1 > /home/rakri/avarhade/DiskANN/prec40M/early-stop-build-x-10-y-0/early_stop_r150_k50.txt +# ./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300 --gt_file /nvmessd1/fbv4/gt100_prec40M --query_file /nvmessd1/fbv4/queries384d.bin --result_path /home/rakri/avarhade/Dump/tmp -K 100 -L 300 -T 1 > /home/rakri/avarhade/DiskANN/prec40M/early-stop-build-x-10-y-0/early_stop_r150_k100.txt +./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_memory_index_r150_l300 --gt_file /nvmessd1/fbv4/gt100_prec40M --query_file /nvmessd1/fbv4/queries384d.bin --result_path /home/rakri/avarhade/Dump/tmp -K 50 -L 50 100 150 200 250 300 -T 1 > /home/rakri/avarhade/DiskANN/prec40M/search_only/early_stop_search_r150_l300_k50_x7_y0_L.txt \ No newline at end of file diff --git a/src/index.cpp b/src/index.cpp index bf93344fa..08056508a 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -4,7 +4,8 @@ #include #include - +#include +#include #include "boost/dynamic_bitset.hpp" #include "index_factory.h" #include "memory_mapper.h" @@ -790,7 +791,7 @@ bool Index::detect_common_filters(uint32_t point_id, bool searc template std::pair Index::iterate_to_fixed_point( - InMemQueryScratch *scratch, const uint32_t Lsize, const std::vector &init_ids, bool use_filter, + InMemQueryScratch *scratch, const uint32_t Lsize, const size_t K, const std::vector &init_ids, bool use_filter, const std::vector &filter_labels, bool search_invocation) { std::vector &expanded_nodes = scratch->pool(); @@ -880,11 +881,44 @@ std::pair Index::iterate_to_fixed_point( uint32_t hops = 0; uint32_t cmps = 0; + std::queue last_insertions; + int x = 7; + int y = 0; + float prev_distance_to_query = 0.0; + while (best_L_nodes.has_unexpanded_node()) - { + { + hops++; + + // Check if the sum of elements of the queue is less than y, if yes then exit the loop + int sum_last_insertions = 0; + std::queue temp_queue = last_insertions; + while (!temp_queue.empty()) { + sum_last_insertions += temp_queue.front(); + temp_queue.pop(); + } + if (sum_last_insertions <= y && hops >= x) { + break; + } + auto nbr = best_L_nodes.closest_unexpanded(); auto n = nbr.id; + if(search_invocation){ + // std::vector id_scratch_temp = {n}; + // std::vector dist_scratch_temp = {0.0}; + // compute_dists(id_scratch_temp, dist_scratch_temp); + // float change_in_distance = std::abs(dist_scratch_temp[0] - prev_distance_to_query); + // if (best_L_nodes.size() == Lsize && (float)(change_in_distance/(dist_scratch_temp[0] + 1e-6)) < 0.01){ + // break; + // } + // prev_distance_to_query = dist_scratch_temp[0]; + // diskann::cout<<"Iteration/Hop: #"< Index::iterate_to_fixed_point( cmps += (uint32_t)id_scratch.size(); // Insert pairs into the pool of candidates + int insertions_this_iter = 0; for (size_t m = 0; m < id_scratch.size(); ++m) - { + { + insertions_this_iter += best_L_nodes.check_in_top_k(Neighbor(id_scratch[m], dist_scratch[m]), K); best_L_nodes.insert(Neighbor(id_scratch[m], dist_scratch[m])); } + last_insertions.push(insertions_this_iter); + if (last_insertions.size() > x) { + last_insertions.pop(); + } } + // if (search_invocation){ + // diskann::cout<<"Total Iterations/Hops: "<expanded_nodes_vec().size()<expanded_nodes_vec()){ + // diskann::cout<::search_for_point_and_prune(int location, uint32_t L if (!use_filter) { _data_store->get_vector(location, scratch->aligned_query()); - iterate_to_fixed_point(scratch, Lindex, init_ids, false, unused_filter_label, false); + iterate_to_fixed_point(scratch, Lindex, Lindex, init_ids, false, unused_filter_label, false); } else { @@ -1002,7 +1051,7 @@ void Index::search_for_point_and_prune(int location, uint32_t L tl.unlock(); _data_store->get_vector(location, scratch->aligned_query()); - iterate_to_fixed_point(scratch, filteredLindex, filter_specific_start_nodes, true, + iterate_to_fixed_point(scratch, filteredLindex, Lindex, filter_specific_start_nodes, true, _location_to_labels[location], false); // combine candidate pools obtained with filter and unfiltered criteria. @@ -1016,7 +1065,7 @@ void Index::search_for_point_and_prune(int location, uint32_t L scratch->clear(); _data_store->get_vector(location, scratch->aligned_query()); - iterate_to_fixed_point(scratch, Lindex, init_ids, false, unused_filter_label, false); + iterate_to_fixed_point(scratch, Lindex, Lindex, init_ids, false, unused_filter_label, false); for (auto unfiltered_neighbour : scratch->pool()) { @@ -1983,7 +2032,7 @@ std::pair Index::search(const T *query, con _data_store->preprocess_query(query, scratch); - auto retval = iterate_to_fixed_point(scratch, L, init_ids, false, unused_filter_label, true); + auto retval = iterate_to_fixed_point(scratch, L, K, init_ids, false, unused_filter_label, true); NeighborPriorityQueue &best_L_nodes = scratch->best_l_nodes(); @@ -2087,7 +2136,7 @@ std::pair Index::search_with_filters(const filter_vec.emplace_back(filter_label); _data_store->preprocess_query(query, scratch); - auto retval = iterate_to_fixed_point(scratch, L, init_ids, true, filter_vec, true); + auto retval = iterate_to_fixed_point(scratch, L, K, init_ids, true, filter_vec, true); auto best_L_nodes = scratch->best_l_nodes(); @@ -2171,14 +2220,14 @@ size_t Index::search_with_tags(const T *query, const uint64_t K if (!use_filters) { const std::vector unused_filter_label; - iterate_to_fixed_point(scratch, L, init_ids, false, unused_filter_label, true); + iterate_to_fixed_point(scratch, L, K, init_ids, false, unused_filter_label, true); } else { std::vector filter_vec; auto converted_label = this->get_converted_label(filter_label); filter_vec.push_back(converted_label); - iterate_to_fixed_point(scratch, L, init_ids, true, filter_vec, true); + iterate_to_fixed_point(scratch, L, K, init_ids, true, filter_vec, true); } NeighborPriorityQueue &best_L_nodes = scratch->best_l_nodes();