[BUG] Brute force python save_load test fail randomly

[rhdong]

The log shows the set of neighbors is correct, but some order of them is not overturned
This could be reproduced with very high possibility by: https://github.com/rhdong/cuvs/tree/rhdong/bf-py-test-fail-reproduce

______________________________________________________________________________ test_save_load_brute_force[35] ______________________________________________________________________________

times = 35

    @pytest.mark.parametrize("times", range(100))
    def test_save_load_brute_force(times):
>       run_save_load(brute_force, np.float32)

python/cuvs/cuvs/tests/test_serialization.py:39: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

ann_module = <module 'cuvs.neighbors.brute_force' from '/root/miniconda/envs/all_cuda-128_arch-x86_64/lib/python3.12/site-packages/cuvs/neighbors/brute_force/__init__.py'>
dtype = <class 'numpy.float32'>

    def run_save_load(ann_module, dtype):
        n_rows = 10000
        n_cols = 50
        n_queries = 100
    
        dataset = generate_data((n_rows, n_cols), dtype)
        dataset_device = device_ndarray(dataset)
    
        if ann_module == brute_force:
            index = ann_module.build(dataset_device)
        else:
            build_params = ann_module.IndexParams()
            index = ann_module.build(build_params, dataset_device)
    
        assert index.trained
        filename = "my_index.bin"
        ann_module.save(filename, index)
        loaded_index = ann_module.load(filename)
    
        queries = generate_data((n_queries, n_cols), dtype)
    
        queries_device = device_ndarray(queries)
        k = 10
        if ann_module == brute_force:
            distance_dev, neighbors_dev = ann_module.search(
                index, queries_device, k
            )
        else:
            search_params = ann_module.SearchParams()
            distance_dev, neighbors_dev = ann_module.search(
                search_params, index, queries_device, k
            )
    
        neighbors = neighbors_dev.copy_to_host()
        dist = distance_dev.copy_to_host()
        del index
    
        if ann_module == brute_force:
            distance_dev, neighbors_dev = ann_module.search(
                loaded_index, queries_device, k
            )
        else:
            distance_dev, neighbors_dev = ann_module.search(
                search_params, loaded_index, queries_device, k
            )
    
        neighbors2 = neighbors_dev.copy_to_host()
        dist2 = distance_dev.copy_to_host()
    
>       assert np.all(neighbors == neighbors2)
E       assert np.False_
E        +  where np.False_ = <function all at 0x73ca2d72d630>(array([[5309,  482, 4361, 7351, 7725, 9483, 4261, 9201, 4898, 2362],\n       [4981, 5384, 9848, 5090, 3101, 9324, 2624, 4378, 5386, 1572],\n       [2482, 7978, 9759, 5434, 8661, 4204, 3381, 6549, 7633,  966],\n       [8305, 3662, 7097, 1067, 5651, 3563, 7178, 7302, 8496, 8716],\n       [1391, 5737, 8254, 2067, 1143, 3594, 3465, 5576, 4158, 2417],\n       [8716, 1114, 7804, 8512, 7235, 2679, 1812, 6336, 4673, 5651],\n       [4499, 9712,  313, 8325, 1815, 4169, 1639, 7929, 6620, 9952],\n       [2268, 8102,  962, 4169, 7842, 8969, 5649, 5228, 2704, 2075],\n       [8983, 9316, 6950, 6374, 4996, 9426,  323, 2174, 3813, 5605],\n       [1601, 2154, 1789, 9006, 4772, 2831, 7864,  515, 8501, 5543],\n       [5287, 1410, 6374, 6182, 4727,  430, 4675, 6491, 1453, 7601],\n       [4015, 4301, 5838, 4924, 6208, 5734, 5661, 2236, 9727, 5767],\n       [8391, 4674, 7221, 4083, 8380, 7830, 5732, 7136, 1006, 6633],\n       [8127, 8408, 1102, 3123, 5236, 2117, 2291, 1327, 6868, 1637],\n       [7426, 7441, 2503, 2872, 5806, 5011,  497, 2560, 5645, 7012],\n       [ 474, 9815, 8658, 5125, 1369, 3851, 1953,  264, 1334, 6398],\n       [7222, 8380, 4884, 1887, 5078, 7613, 3378, 6246, 6868, 7940],\n       [2323, 9748, 8667, 9541, 9650, 6830, 5994, 9856, 9013, 8891],\n       [3129, 9090, 8938, 3994, 8380, 2799,   24, 8734, 9047, 8921],\n       [7049,  248, 4904, 8070, 1476, 4775, 2307, 5836, 9970, 2174],\n       [7715, 2898, 7035, 5499,  844, 7945, 2257, 6113, 1851, 5479],\n       [4754, 8973, 4283, 9839, 1332, 6818,  840, 8102, 2602, 6837],\n       [7946, 6017, 5214, 7111, 7626, 7029,  303, 1196, 1228, 4519],\n       [ 315, 7405,  697, 5816, 4730, 3451, 7631, 9621, 4918, 1318],\n       [6663, 2052, 4277, 6351, 8888, 1234, 7688, 1699, 9446, 4108],\n       [9902, 7538, 6438, 1894, 6812, 7357,  121, 8741, 7830, 5253],\n       [6631, 5497, 7199, 2170, 8169, 8686, 4918, 8842, 8023, 9426],\n       [3906, 7736,  342, 3078, 9388, 5407, 7325, 1155, 3382, 5434],\n       [3357, 1679, 6360, 7962, 4016, 2773, 2464, 7214, 2964,  462],\n       [1332,  413,   55, 2686, 6201, 7715, 3600, 7069, 8894, 5644],\n       [ 729,   99, 8647, 2975, 6617, 4530, 2217, 1665, 3249, 3390],\n       [4243, 5090, 5674, 4691,   39, 4924, 4544, 8269, 8858, 8468],\n       [8969,  604, 2447, 6939, 5390, 8318, 2595, 5994, 1564, 3491],\n       [5214, 7476, 9216, 5648, 1228, 5115, 7626, 9480, 2146, 6938],\n       [7051, 1785,  514, 8526,  677, 2527, 2498, 6210, 3553, 8588],\n       [8747, 9471, 7046, 1145,  571, 6024, 6843, 4633,  341, 2536],\n       [9835, 7564, 7260, 7779, 3763,  968, 4352, 5221, 6874, 3192],\n       [7669, 9294, 2535,  635, 1018, 9339, 4226, 4772, 3887, 4644],\n       [1977, 4051, 3364, 4511,   38, 2631, 9707, 7199, 9745, 6638],\n       [5667,  717, 5818,   52, 6173, 2462, 1049,   69, 6902, 7257],\n       [3993, 8469, 1507, 8733, 3009, 7999, 4264, 5045, 5650, 6696],\n       [4327,  439, 1476, 1587, 9827, 7540, 4242, 2452, 5842, 9502],\n       [3110, 1131, 4526, 7174, 1467, 7384, 6910, 6603, 6671, 7052],\n       [9784, 3756, 8625, 1990, 7370, 9850, 2159, 1115, 8401,  614],\n       [7043, 1153,  288, 5316, 4741, 1966, 4600, 8583, 1339, 4457],\n       [8942, 8403, 2097, 7750,  184, 5522, 1158, 1948, 5911, 4078],\n       [6308, 9986, 6347, 3915, 6209, 5384, 4953, 9341, 1357, 4537],\n       [6204, 2059, 3940, 6602, 5368, 6681, 1582, 3657, 8185, 3186],\n       [9888, 9023, 6986, 7768, 5047, 2319, 1008, 6191, 2741, 3936],\n       [6237, 7473, 1125,  935, 5187, 7976, 7695, 2807, 8888,  890],\n       [6253, 1328, 6657,  783, 4729, 2747, 6477, 5128, 4549, 6106],\n       [7520, 5240, 5504,  918, 2412, 2576, 6318, 6483, 7113, 7178],\n       [1644, 7364, 5178, 9544, 4114, 7429, 1407, 2188, 3503,  711],\n       [8855,  641, 2711, 8607, 7129, 2303, 9925, 9223, 1216, 6915],\n       [8018, 1468, 6614, 8574, 7736,  121, 8464, 6890, 6352, 6297],\n       [9837, 5977, 3350, 8988, 4289, 6516, 9991, 7120, 6129, 8167],\n       [9419, 6510, 9299, 2590,  423,  686, 7182, 6276, 4270, 4858],\n       [2045, 9160, 2781, 1004, 8250, 8200, 3973, 8451, 3798, 4403],\n       [7252, 1660, 2672, 6825, 9330, 2437, 9821, 9011, 3451, 6748],\n       [2416, 4038, 8451, 9926,  216, 8196, 9893, 7604, 5451, 7700],\n       [8644, 9309, 5251, 3521, 7980, 9907,  660, 3676, 5949, 3670],\n       [2416, 3282, 6775, 8519, 9761, 8952, 5787, 4472, 8723, 5912],\n       [7339, 8824, 8016,  598, 7817, 1730,  898, 5559, 5661, 2662],\n       [9173, 7370,  271, 6007, 7915, 9425, 3357, 9176, 9572, 5913],\n       [8169,   69, 2690, 1902, 8423, 5739, 7349, 4544, 7928, 8369],\n       [8530,  301, 7370, 9560, 6614, 6575, 8033,  841,  248, 5709],\n       [5019, 7187, 4531, 5096, 2642,  526, 8595, 3665, 1330, 4929],\n       [4038, 5030, 4756, 7726, 2193, 8451, 3907, 3910,   41, 7901],\n       [6175, 7370, 4460, 8271, 9050, 3690, 9743, 2497, 8380, 7705],\n       [9361,  763, 6102, 2067, 2426, 8423, 5904, 2146, 4247, 6753],\n       [ 807, 2849, 3887, 4347, 8870,  831, 9151, 5947, 3193, 6109],\n       [3686, 9889, 8201, 8897,  816, 2991, 6246, 5979, 3546, 6890],\n       [ 230, 4558, 4537, 8890, 9335, 9754, 2527, 5384, 6203, 2360],\n       [ 155, 6761, 1422, 9340, 6309,  316, 4268, 7517,  661, 9658],\n       [8988, 4554, 4261, 9294, 4557, 4297, 5434, 1030, 4848, 3057],\n       [2848, 6525, 6780, 9991, 1751, 8644, 5225, 6414, 1190, 8747],\n       [9712, 1880, 7405, 6218, 9821, 8744, 8128, 2751, 3301, 7817],\n       [2230, 9014, 2576, 1032, 9011, 1440,  540,  112, 7280,  998],\n       [ 655, 2135, 8204, 5315, 2873, 3908, 9817, 5834,  510, 7374],\n       [8103,   90, 9348, 6847, 5726, 1882, 9848,  684, 4293, 5848],\n       [7689, 8549, 9826,  979, 5539, 2373, 2444, 9877, 8131, 2308],\n       [5478, 2171, 6930, 7273, 5489, 4851, 6174, 5496, 3438, 7169],\n       [1051, 5938, 2729, 9821, 7773, 7255, 9148, 3779, 6061, 2679],\n       [2820, 1219, 8871, 4391, 6201, 6085, 9943, 8671, 6939, 5401],\n       [7170, 2333, 9967, 8076, 2453, 6971,  891, 8188,  717, 3675],\n       [3980, 4956, 7026, 9431, 3621, 8993, 4754, 7205, 4347, 2008],\n       [4608, 3652,  420, 2082, 3811, 6339, 8121,  429, 5001, 8369],\n       [6534, 3447,  960, 7564, 8112,  568, 8866, 4591, 3255, 2807],\n       [6469, 7830, 8169, 2394, 2107, 7089, 8318, 6608, 2614, 7778],\n       [2656, 5810, 2417, 5783, 9983, 6544, 3907, 4989, 3029, 8353],\n       [8050, 2135,  206, 7872, 6841, 2324, 3987, 3000, 3180, 1617],\n       [2972,  993, 2942, 2828,   63, 7738, 1901, 5082, 1184, 3065],\n       [8672, 8139, 9002, 8456, 5276, 6939, 1131, 4727, 9524, 4828],\n       [9902,  525, 7736, 7071, 3625, 6808, 9131, 3410, 8864, 9546],\n       [7582, 6601, 2089, 4367, 4826, 5071, 6047, 2222, 5328, 2606],\n       [3854, 6909, 1553, 3207, 1500, 3616, 8927, 1843, 8511, 4840],\n       [6535, 9014, 3972,  836, 8565, 6944, 3718, 4385, 3080, 5393],\n       [2084, 6051, 4382, 2935,  676, 9946, 8957, 2191, 3147, 4436],\n       [8508, 7221,  742, 2670, 3689, 9877, 6054, 6463, 5841, 8271],\n       [8892, 6046, 5386,  904, 7780, 8908, 9881, 5008, 7156, 1508]]) == array([[5309,  482, 4361, 7351, 7725, 9483, 4261, 9201, 4898, 2362],\n       [4981, 5384, 9848, 5090, 3101, 9324, 2624, 4378, 5386, 1572],\n       [2482, 7978, 9759, 5434, 8661, 4204, 3381, 6549, 7633,  966],\n       [8305, 3662, 7097, 1067, 5651, 3563, 7178, 7302, 8496, 8716],\n       [1391, 5737, 8254, 2067, 1143, 3594, 3465, 5576, 4158, 2417],\n       [8716, 1114, 7804, 8512, 7235, 2679, 1812, 6336, 4673, 5651],\n       [4499, 9712,  313, 8325, 1815, 4169, 1639, 7929, 6620, 9952],\n       [2268, 8102,  962, 4169, 7842, 8969, 5649, 5228, 2704, 2075],\n       [8983, 9316, 6950, 6374, 4996, 9426,  323, 2174, 3813, 5605],\n       [1601, 2154, 1789, 9006, 4772, 2831, 7864,  515, 8501, 5543],\n       [5287, 1410, 6374, 6182, 4727,  430, 4675, 6491, 1453, 7601],\n       [4015, 4301, 5838, 4924, 6208, 5734, 5661, 2236, 9727, 5767],\n       [8391, 4674, 7221, 4083, 8380, 7830, 5732, 7136, 1006, 6633],\n       [8127, 8408, 1102, 3123, 5236, 2117, 2291, 1327, 6868, 1637],\n       [7426, 7441, 2503, 2872, 5806, 5011,  497, 2560, 5645, 7012],\n       [ 474, 9815, 8658, 5125, 1369, 3851, 1953,  264, 1334, 6398],\n       [7222, 8380, 4884, 1887, 5078, 7613, 3378, 6246, 6868, 7940],\n       [2323, 9748, 8667, 9541, 9650, 6830, 5994, 9856, 9013, 8891],\n       [3129, 9090, 8938, 3994, 8380, 2799,   24, 8734, 9047, 8921],\n       [7049,  248, 4904, 8070, 1476, 4775, 2307, 5836, 9970, 2174],\n       [7715, 2898, 7035, 5499,  844, 7945, 2257, 6113, 1851, 5479],\n       [4754, 8973, 4283, 9839, 1332, 6818,  840, 8102, 2602, 6837],\n       [7946, 6017, 5214, 7111, 7626, 7029,  303, 1196, 1228, 4519],\n       [ 315, 7405,  697, 5816, 4730, 3451, 7631, 9621, 4918, 1318],\n       [6663, 2052, 4277, 6351, 8888, 1234, 7688, 1699, 9446, 4108],\n       [9902, 7538, 6438, 1894, 6812, 7357,  121, 8741, 7830, 5253],\n       [6631, 5497, 7199, 2170, 8169, 8686, 4918, 8842, 8023, 9426],\n       [3906, 7736,  342, 3078, 9388, 5407, 7325, 1155, 3382, 5434],\n       [3357, 1679, 6360, 7962, 4016, 2773, 2464, 7214, 2964,  462],\n       [1332,  413,   55, 2686, 6201, 7715, 3600, 7069, 8894, 5644],\n       [ 729,   99, 8647, 2975, 6617, 4530, 2217, 1665, 3249, 3390],\n       [4243, 5090, 5674, 4691,   39, 4924, 4544, 8269, 8858, 8468],\n       [8969,  604, 2447, 6939, 5390, 8318, 2595, 5994, 1564, 3491],\n       [5214, 7476, 9216, 5648, 1228, 5115, 7626, 9480, 2146, 6938],\n       [7051, 1785,  514, 8526,  677, 2527, 2498, 6210, 3553, 8588],\n       [8747, 9471, 7046, 1145,  571, 6024, 6843, 4633,  341, 2536],\n       [9835, 7564, 7260, 7779, 3763,  968, 4352, 5221, 6874, 3192],\n       [7669, 9294, 2535,  635, 1018, 9339, 4226, 4772, 3887, 4644],\n       [1977, 4051, 3364, 4511,   38, 2631, 9707, 7199, 9745, 6638],\n       [5667,  717, 5818,   52, 6173, 2462, 1049,   69, 6902, 7257],\n       [3993, 8469, 1507, 8733, 3009, 7999, 4264, 5045, 5650, 6696],\n       [4327,  439, 1476, 1587, 9827, 7540, 4242, 2452, 5842, 9502],\n       [3110, 1131, 4526, 7174, 1467, 7384, 6910, 6603, 6671, 7052],\n       [9784, 3756, 8625, 1990, 7370, 9850, 2159, 1115, 8401,  614],\n       [7043, 1153,  288, 5316, 4741, 1966, 4600, 8583, 1339, 4457],\n       [8942, 8403, 2097, 7750,  184, 5522, 1158, 1948, 5911, 4078],\n       [6308, 9986, 6347, 3915, 6209, 5384, 4953, 9341, 1357, 4537],\n       [6204, 2059, 3940, 6602, 5368, 6681, 1582, 3657, 8185, 3186],\n       [9888, 9023, 6986, 7768, 5047, 2319, 1008, 6191, 2741, 3936],\n       [6237, 7473, 1125,  935, 5187, 7976, 7695, 2807, 8888,  890],\n       [6253, 1328, 6657,  783, 4729, 2747, 6477, 5128, 4549, 6106],\n       [7520, 5240, 5504,  918, 2412, 2576, 6318, 6483, 7113, 7178],\n       [1644, 7364, 5178, 9544, 4114, 7429, 1407, 2188, 3503,  711],\n       [8855,  641, 2711, 8607, 7129, 2303, 9925, 9223, 1216, 6915],\n       [8018, 1468, 6614, 8574, 7736,  121, 8464, 6890, 6352, 6297],\n       [9837, 5977, 3350, 8988, 4289, 6516, 9991, 7120, 6129, 8167],\n       [9419, 6510, 9299, 2590,  423,  686, 7182, 6276, 4270, 4858],\n       [2045, 9160, 2781, 1004, 8250, 8200, 3973, 8451, 3798, 4403],\n       [7252, 1660, 2672, 6825, 9330, 2437, 9821, 9011, 3451, 6748],\n       [2416, 4038, 8451, 9926,  216, 8196, 9893, 7604, 5451, 7700],\n       [8644, 9309, 5251, 3521, 7980, 9907,  660, 3676, 5949, 3670],\n       [2416, 3282, 6775, 8519, 9761, 8952, 5787, 4472, 8723, 5912],\n       [7339, 8824, 8016,  598, 7817, 1730,  898, 5559, 5661, 2662],\n       [9173, 7370,  271, 6007, 7915, 9425, 3357, 9176, 9572, 5913],\n       [8169,   69, 2690, 1902, 8423, 5739, 7349, 4544, 7928, 8369],\n       [8530,  301, 7370, 9560, 6614, 6575, 8033,  841,  248, 5709],\n       [5019, 7187, 4531, 5096, 2642,  526, 8595, 3665, 1330, 4929],\n       [4038, 5030, 4756, 7726, 2193, 8451, 3907, 3910,   41, 7901],\n       [6175, 7370, 4460, 8271, 9050, 3690, 9743, 2497, 8380, 7705],\n       [9361,  763, 6102, 2067, 2426, 8423, 5904, 2146, 4247, 6753],\n       [ 807, 2849, 3887, 4347, 8870,  831, 9151, 5947, 3193, 6109],\n       [3686, 9889, 8201, 8897,  816, 2991, 6246, 5979, 3546, 6890],\n       [ 230, 4558, 4537, 8890, 9335, 9754, 2527, 5384, 6203, 2360],\n       [ 155, 6761, 1422, 9340, 6309,  316, 4268, 7517,  661, 9658],\n       [8988, 4554, 4261, 9294, 4557, 4297, 5434, 1030, 4848, 3057],\n       [2848, 6525, 6780, 9991, 1751, 8644, 5225, 6414, 1190, 8747],\n       [9712, 1880, 7405, 6218, 9821, 8744, 8128, 2751, 3301, 7817],\n       [2230, 9014, 2576, 1032, 9011, 1440,  540,  112, 7280,  998],\n       [ 655, 2135, 8204, 5315, 2873, 3908, 9817, 5834,  510, 7374],\n       [8103,   90, 9348, 6847, 5726, 1882, 9848,  684, 4293, 5848],\n       [7689, 8549, 9826,  979, 5539, 2373, 2444, 9877, 8131, 2308],\n       [5478, 2171, 6930, 7273, 5489, 4851, 6174, 5496, 3438, 7169],\n       [1051, 5938, 2729, 9821, 7773, 7255, 9148, 3779, 6061, 2679],\n       [2820, 1219, 8871, 4391, 6201, 6085, 9943, 8671, 6939, 5401],\n       [7170, 2333, 9967, 8076, 2453, 6971,  891, 8188,  717, 3675],\n       [3980, 4956, 7026, 9431, 3621, 8993, 4754, 7205, 4347, 2008],\n       [4608, 3652,  420, 2082, 3811, 6339, 8121,  429, 5001, 8369],\n       [6534, 3447,  960, 7564, 8112,  568, 8866, 4591, 3255, 2807],\n       [6469, 7830, 8169, 2394, 2107, 7089, 8318, 6608, 2614, 7778],\n       [2656, 5810, 2417, 5783, 9983, 6544, 3907, 4989, 3029, 8353],\n       [8050, 2135,  206, 7872, 6841, 2324, 3987, 3000, 3180, 1617],\n       [2972,  993, 2942, 2828,   63, 7738, 1901, 5082, 1184, 3065],\n       [8672, 8139, 9002, 8456, 5276, 6939, 1131, 4727, 9524, 4828],\n       [9902,  525, 7736, 7071, 3625, 6808, 9131, 3410, 8864, 9546],\n       [7582, 6601, 2089, 4367, 4826, 5071, 6047, 2222, 5328, 2606],\n       [3854, 6909, 1553, 3207, 1500, 3616, 8927, 1843, 8511, 4840],\n       [6535, 9014, 3972,  836, 8565, 6944, 3718, 4385, 3080, 5393],\n       [2084, 6051, 4382, 2935,  676, 9946, 8957, 2191, 3147, 4436],\n       [8508, 7221,  742, 2670, 3689, 9877, 6054, 6463, 8271, 5841],\n       [8892, 6046, 5386,  904, 7780, 8908, 9881, 5008, 7156, 1508]])
E        +    where <function all at 0x73ca2d72d630> = np.all
E           Full diff:
E             array([[5309,  482, 4361, 7351, 7725, 9483, 4261, 9201, 4898, 2362],
E                    [4981, 5384, 9848, 5090, 3101, 9324, 2624, 4378, 5386, 1572],
E                    [2482, 7978, 9759, 5434, 8661, 4204, 3381, 6549, 7633,  966],
E                    [8305, 3662, 7097, 1067, 5651, 3563, 7178, 7302, 8496, 8716],
E                    [1391, 5737, 8254, 2067, 1143, 3594, 3465, 5576, 4158, 2417],
E                    [8716, 1114, 7804, 8512, 7235, 2679, 1812, 6336, 4673, 5651],
E                    [4499, 9712,  313, 8325, 1815, 4169, 1639, 7929, 6620, 9952],
E                    [2268, 8102,  962, 4169, 7842, 8969, 5649, 5228, 2704, 2075],
E                    [8983, 9316, 6950, 6374, 4996, 9426,  323, 2174, 3813, 5605],
E                    [1601, 2154, 1789, 9006, 4772, 2831, 7864,  515, 8501, 5543],
E                    [5287, 1410, 6374, 6182, 4727,  430, 4675, 6491, 1453, 7601],
E                    [4015, 4301, 5838, 4924, 6208, 5734, 5661, 2236, 9727, 5767],
E                    [8391, 4674, 7221, 4083, 8380, 7830, 5732, 7136, 1006, 6633],
E                    [8127, 8408, 1102, 3123, 5236, 2117, 2291, 1327, 6868, 1637],
E                    [7426, 7441, 2503, 2872, 5806, 5011,  497, 2560, 5645, 7012],
E                    [ 474, 9815, 8658, 5125, 1369, 3851, 1953,  264, 1334, 6398],
E                    [7222, 8380, 4884, 1887, 5078, 7613, 3378, 6246, 6868, 7940],
E                    [2323, 9748, 8667, 9541, 9650, 6830, 5994, 9856, 9013, 8891],
E                    [3129, 9090, 8938, 3994, 8380, 2799,   24, 8734, 9047, 8921],
E                    [7049,  248, 4904, 8070, 1476, 4775, 2307, 5836, 9970, 2174],
E                    [7715, 2898, 7035, 5499,  844, 7945, 2257, 6113, 1851, 5479],
E                    [4754, 8973, 4283, 9839, 1332, 6818,  840, 8102, 2602, 6837],
E                    [7946, 6017, 5214, 7111, 7626, 7029,  303, 1196, 1228, 4519],
E                    [ 315, 7405,  697, 5816, 4730, 3451, 7631, 9621, 4918, 1318],
E                    [6663, 2052, 4277, 6351, 8888, 1234, 7688, 1699, 9446, 4108],
E                    [9902, 7538, 6438, 1894, 6812, 7357,  121, 8741, 7830, 5253],
E                    [6631, 5497, 7199, 2170, 8169, 8686, 4918, 8842, 8023, 9426],
E                    [3906, 7736,  342, 3078, 9388, 5407, 7325, 1155, 3382, 5434],
E                    [3357, 1679, 6360, 7962, 4016, 2773, 2464, 7214, 2964,  462],
E                    [1332,  413,   55, 2686, 6201, 7715, 3600, 7069, 8894, 5644],
E                    [ 729,   99, 8647, 2975, 6617, 4530, 2217, 1665, 3249, 3390],
E                    [4243, 5090, 5674, 4691,   39, 4924, 4544, 8269, 8858, 8468],
E                    [8969,  604, 2447, 6939, 5390, 8318, 2595, 5994, 1564, 3491],
E                    [5214, 7476, 9216, 5648, 1228, 5115, 7626, 9480, 2146, 6938],
E                    [7051, 1785,  514, 8526,  677, 2527, 2498, 6210, 3553, 8588],
E                    [8747, 9471, 7046, 1145,  571, 6024, 6843, 4633,  341, 2536],
E                    [9835, 7564, 7260, 7779, 3763,  968, 4352, 5221, 6874, 3192],
E                    [7669, 9294, 2535,  635, 1018, 9339, 4226, 4772, 3887, 4644],
E                    [1977, 4051, 3364, 4511,   38, 2631, 9707, 7199, 9745, 6638],
E                    [5667,  717, 5818,   52, 6173, 2462, 1049,   69, 6902, 7257],
E                    [3993, 8469, 1507, 8733, 3009, 7999, 4264, 5045, 5650, 6696],
E                    [4327,  439, 1476, 1587, 9827, 7540, 4242, 2452, 5842, 9502],
E                    [3110, 1131, 4526, 7174, 1467, 7384, 6910, 6603, 6671, 7052],
E                    [9784, 3756, 8625, 1990, 7370, 9850, 2159, 1115, 8401,  614],
E                    [7043, 1153,  288, 5316, 4741, 1966, 4600, 8583, 1339, 4457],
E                    [8942, 8403, 2097, 7750,  184, 5522, 1158, 1948, 5911, 4078],
E                    [6308, 9986, 6347, 3915, 6209, 5384, 4953, 9341, 1357, 4537],
E                    [6204, 2059, 3940, 6602, 5368, 6681, 1582, 3657, 8185, 3186],
E                    [9888, 9023, 6986, 7768, 5047, 2319, 1008, 6191, 2741, 3936],
E                    [6237, 7473, 1125,  935, 5187, 7976, 7695, 2807, 8888,  890],
E                    [6253, 1328, 6657,  783, 4729, 2747, 6477, 5128, 4549, 6106],
E                    [7520, 5240, 5504,  918, 2412, 2576, 6318, 6483, 7113, 7178],
E                    [1644, 7364, 5178, 9544, 4114, 7429, 1407, 2188, 3503,  711],
E                    [8855,  641, 2711, 8607, 7129, 2303, 9925, 9223, 1216, 6915],
E                    [8018, 1468, 6614, 8574, 7736,  121, 8464, 6890, 6352, 6297],
E                    [9837, 5977, 3350, 8988, 4289, 6516, 9991, 7120, 6129, 8167],
E                    [9419, 6510, 9299, 2590,  423,  686, 7182, 6276, 4270, 4858],
E                    [2045, 9160, 2781, 1004, 8250, 8200, 3973, 8451, 3798, 4403],
E                    [7252, 1660, 2672, 6825, 9330, 2437, 9821, 9011, 3451, 6748],
E                    [2416, 4038, 8451, 9926,  216, 8196, 9893, 7604, 5451, 7700],
E                    [8644, 9309, 5251, 3521, 7980, 9907,  660, 3676, 5949, 3670],
E                    [2416, 3282, 6775, 8519, 9761, 8952, 5787, 4472, 8723, 5912],
E                    [7339, 8824, 8016,  598, 7817, 1730,  898, 5559, 5661, 2662],
E                    [9173, 7370,  271, 6007, 7915, 9425, 3357, 9176, 9572, 5913],
E                    [8169,   69, 2690, 1902, 8423, 5739, 7349, 4544, 7928, 8369],
E                    [8530,  301, 7370, 9560, 6614, 6575, 8033,  841,  248, 5709],
E                    [5019, 7187, 4531, 5096, 2642,  526, 8595, 3665, 1330, 4929],
E                    [4038, 5030, 4756, 7726, 2193, 8451, 3907, 3910,   41, 7901],
E                    [6175, 7370, 4460, 8271, 9050, 3690, 9743, 2497, 8380, 7705],
E                    [9361,  763, 6102, 2067, 2426, 8423, 5904, 2146, 4247, 6753],
E                    [ 807, 2849, 3887, 4347, 8870,  831, 9151, 5947, 3193, 6109],
E                    [3686, 9889, 8201, 8897,  816, 2991, 6246, 5979, 3546, 6890],
E                    [ 230, 4558, 4537, 8890, 9335, 9754, 2527, 5384, 6203, 2360],
E                    [ 155, 6761, 1422, 9340, 6309,  316, 4268, 7517,  661, 9658],
E                    [8988, 4554, 4261, 9294, 4557, 4297, 5434, 1030, 4848, 3057],
E                    [2848, 6525, 6780, 9991, 1751, 8644, 5225, 6414, 1190, 8747],
E                    [9712, 1880, 7405, 6218, 9821, 8744, 8128, 2751, 3301, 7817],
E                    [2230, 9014, 2576, 1032, 9011, 1440,  540,  112, 7280,  998],
E                    [ 655, 2135, 8204, 5315, 2873, 3908, 9817, 5834,  510, 7374],
E                    [8103,   90, 9348, 6847, 5726, 1882, 9848,  684, 4293, 5848],
E                    [7689, 8549, 9826,  979, 5539, 2373, 2444, 9877, 8131, 2308],
E                    [5478, 2171, 6930, 7273, 5489, 4851, 6174, 5496, 3438, 7169],
E                    [1051, 5938, 2729, 9821, 7773, 7255, 9148, 3779, 6061, 2679],
E                    [2820, 1219, 8871, 4391, 6201, 6085, 9943, 8671, 6939, 5401],
E                    [7170, 2333, 9967, 8076, 2453, 6971,  891, 8188,  717, 3675],
E                    [3980, 4956, 7026, 9431, 3621, 8993, 4754, 7205, 4347, 2008],
E                    [4608, 3652,  420, 2082, 3811, 6339, 8121,  429, 5001, 8369],
E                    [6534, 3447,  960, 7564, 8112,  568, 8866, 4591, 3255, 2807],
E                    [6469, 7830, 8169, 2394, 2107, 7089, 8318, 6608, 2614, 7778],
E                    [2656, 5810, 2417, 5783, 9983, 6544, 3907, 4989, 3029, 8353],
E                    [8050, 2135,  206, 7872, 6841, 2324, 3987, 3000, 3180, 1617],
E                    [2972,  993, 2942, 2828,   63, 7738, 1901, 5082, 1184, 3065],
E                    [8672, 8139, 9002, 8456, 5276, 6939, 1131, 4727, 9524, 4828],
E                    [9902,  525, 7736, 7071, 3625, 6808, 9131, 3410, 8864, 9546],
E                    [7582, 6601, 2089, 4367, 4826, 5071, 6047, 2222, 5328, 2606],
E                    [3854, 6909, 1553, 3207, 1500, 3616, 8927, 1843, 8511, 4840],
E                    [6535, 9014, 3972,  836, 8565, 6944, 3718, 4385, 3080, 5393],
E                    [2084, 6051, 4382, 2935,  676, 9946, 8957, 2191, 3147, 4436],
E           -        [8508, 7221,  742, 2670, 3689, 9877, 6054, 6463, 8271, 5841],
E           ?                                                             ------
E           +        [8508, 7221,  742, 2670, 3689, 9877, 6054, 6463, 5841, 8271],
E           ?                                                         ++++++
E                    [8892, 6046, 5386,  904, 7780, 8908, 9881, 5008, 7156, 1508]],
E             ))

python/cuvs/cuvs/tests/test_serialization.py:91: AssertionError
================================================================================= short test summary info ==================================================================================
FAILED python/cuvs/cuvs/tests/test_serialization.py::test_save_load_brute_force[12] - assert np.False_
FAILED python/cuvs/cuvs/tests/test_serialization.py::test_save_load_brute_force[32] - assert np.False_
FAILED python/cuvs/cuvs/tests/test_serialization.py::test_save_load_brute_force[35] - assert np.False_
=============================================================================== 3 failed, 97 passed in 0

[rhdong]

I found one query distances with different to samples are very close occasionally:

E             array([[7356, 6315, 3579, 2860, 3790, 1747, 9880, 2584, 4067, 2456],
E                    [7520, 6982, 5815,  929, 2325, 2250, 1439, 9866, 8289, 4179],
E                    [5404, 6802, 2551, 5991, 1892, 1024, 2428, 6699, 2606, 7739],
E                    [5774, 8981, 6457, 7510, 8410, 9036, 9010, 2026, 2463, 8314],
E                    [1369, 9634,  338, 8415, 9194, 3971, 1448, 9331, 1082, 1115],
E                    [7754, 1067, 7113, 3789, 1614, 6765, 7434,  541, 8518, 2720],
E                    [8104, 8804, 6654, 1776, 4759, 4700, 9157,  744, 9911, 9321],
E                    [8735, 3937, 5238, 5307, 9597, 4964, 8644, 3688, 3545, 8054],
E                    [5948, 1218, 8975, 3946, 7593, 2818, 9065, 1631, 5087, 9053],
E                    [6484, 8709, 7806, 9591, 6222, 3882, 3098, 5923, 1283, 2439],
E                    [8394, 3987, 7022, 7663, 2785, 4737, 2056, 4049, 8812, 7542],
E                    [4657, 7893,  332, 8275, 6701, 6561, 3746, 2839, 9894, 5603],
E                    [8009, 2137, 2013, 1234, 9554,  249, 4547, 9061, 8111, 6968],
E                    [5326,  512, 9282, 4583, 3257,  654, 8371, 7634, 2745, 3021],
E                    [ 543, 6044, 5574, 4648, 3479, 7515, 2366, 8803, 5344, 7147],
E                    [5148, 3850, 4997, 2023, 2537, 2392, 3603, 8942, 8239, 3979],
E                    [4257, 4958, 3282, 1352, 3671, 1301, 1036, 3618, 4553, 3117],
E                    [4014, 7698, 4968, 7374, 6330, 3304, 3840, 2815, 4923, 9442],
E                    [7005, 1510, 9989, 9899, 5163, 7446, 9105, 6494, 8157, 5861],
E                    [ 593, 5258, 3633, 2708, 7588, 2196, 7364, 7779, 7036, 5612],
E                    [2448, 1863,  100, 3025, 1347, 4543, 7865, 4251,  251, 6939],
E                    [ 339, 7729, 8217, 5146, 5502, 2968, 8170, 7342, 5736, 3631],
E                    [9443, 9989, 5761, 9708, 9475, 9838, 2443,  641, 6118, 4325],
E                    [6073, 1128, 2027, 1558, 8969,   59, 7697, 3103, 4433, 7554],
E                    [4006, 9111, 8843, 4222, 8114,  509,  402, 3178, 5061, 1806],
E                    [ 609, 7838, 5915, 8270,  871, 9903, 7710, 8054,   29, 2906],
E                    [5312, 9984, 1499, 6424, 8703, 3908, 1291, 6405, 9563, 2469],
E                    [8868, 6144, 8657, 6008, 2755, 2112, 6999, 2860, 5169, 4849],
E                    [9227, 6329, 7620, 1352, 7751, 8577, 5430, 4524, 8615, 8882],
E                    [6497, 6595, 3627, 7979, 1118, 3082, 3075, 1519, 7416, 9992],
E                    [4499, 7839, 3740, 6926, 2839, 5849, 4634, 9760, 6142, 9553],
E                    [6603, 2519, 4735, 2744, 6931, 9085, 4633, 8498, 2755, 4033],
E                    [ 175, 2110,    7, 1412, 1798, 1481, 8614, 1371,  910, 4410],
E                    [5100, 9721, 7463, 6862, 5599, 2337, 7774, 1352, 8642, 9823],
E                    [7126, 6686, 5524, 5590, 3049, 1207, 1657, 1299, 8063, 1240],
E                    [4630, 4522, 6362, 3991, 4703,  404, 4806, 3987, 8907, 2668],
E                    [1806, 3429, 5282, 2632, 9513, 6054, 1176, 2030, 8223, 4034],
E                    [1682, 2312, 8311, 4145, 2564, 9377, 9194, 7276, 6398, 8011],
E                    [3354, 2311, 3432, 3160, 8459, 2196, 2540, 7471, 9730,  346],
E                    [5835, 4479, 2361,  398, 8541, 2573, 4455, 9601, 6844, 2813],
E                    [8434, 2691, 8829, 7797, 9241,  119, 2212, 7958, 5488, 5151],
E                    [5737, 9222, 2020, 1136, 4479, 2518, 9206, 3687, 4255,  107],
E                    [6184, 6582, 3734, 3876, 9532,  730, 8907, 7171, 3682, 7437],
E                    [3581, 9124, 3998, 3021, 3323, 2810, 7940, 6185, 7069, 9403],
E           -        [8882, 9900, 8757,  107, 8479, 5171, 9484, 9775, 3933, 2111],
E           ?                                             ------
E           +        [8882, 9900, 8757,  107, 8479, 9484, 5171, 9775, 3933, 2111],
E           ?                                       ++++++                                 ++++++

 [3.3649712 3.8245468 4.1913643 4.262558  4.3329716 4.3587875 4.3587875
  4.4026165 4.408066  4.4113426]

It can be reproduced by the code:

import numpy as np

dataset_5171 = np.array( [9.32142138e-01, 8.72813582e-01, 7.71189928e-01, 5.85802495e-01,
  2.38001600e-01, 7.06278622e-01, 8.78981054e-01, 4.03263688e-01,
  4.75592285e-01, 1.45135760e-01, 3.21364909e-01, 6.87913895e-01,
  8.30989003e-01, 2.19078675e-01, 6.42280996e-01, 4.20177460e-01,
  8.86989295e-01, 3.68644297e-01, 9.93755877e-01, 6.99248761e-02,
  9.09213126e-02, 2.24599376e-01, 8.28429461e-01, 3.59329879e-01,
  5.71323633e-01, 6.59366027e-02, 4.18484896e-01, 8.31805766e-01,
  2.35715076e-01, 8.27406108e-01, 8.21960211e-01, 4.06000704e-01,
  4.96624112e-01, 4.26898181e-01, 2.14131534e-01, 6.58396363e-01,
  3.63032669e-01, 9.13158238e-01, 8.36311519e-01, 2.79704154e-01,
  4.96733159e-01, 9.39586386e-02, 8.37650478e-01, 7.54839361e-01,
  4.97722834e-01, 5.02949297e-01, 8.90806139e-01, 5.37597716e-01,
  7.28471994e-01, 7.22921133e-01], dtype=float)

dataset_9484 = np.array( [9.81457651e-01, 5.93166947e-01, 1.56116426e-01, 7.87705481e-01,
  6.65047050e-01, 6.26070380e-01, 1.89543471e-01, 8.99864018e-01,
  4.08117533e-01, 2.98274849e-02, 8.38999808e-01, 9.41052794e-01,
  7.70968139e-01, 1.60131723e-01, 9.21209812e-01, 4.37662721e-01,
  7.82714367e-01, 9.02948081e-01, 9.07859057e-02, 4.08284068e-01,
  4.53266650e-01, 6.05524331e-03, 9.13958311e-01, 7.49397278e-01,
  4.31984991e-01, 1.09489582e-01, 4.56626981e-01, 3.62299412e-01,
  1.01631679e-01, 6.10530853e-01, 9.31025982e-01, 7.03031659e-01,
  4.18992788e-01, 7.79154778e-01, 6.04481623e-03, 1.96646154e-01,
  8.79877806e-01, 4.18404818e-01, 1.33909434e-01, 6.40034139e-01,
  6.06465399e-01, 2.39080772e-01, 7.01063633e-01, 7.51568615e-01,
  9.95701730e-01, 5.62685132e-01, 4.60158437e-01, 2.16199681e-01,
  4.73403454e-01, 6.09784663e-01], dtype=float)

query = np.array([
    6.98716998e-01, 7.96642601e-01, 4.80482608e-01, 4.85115111e-01,
    4.14034247e-01, 6.72952533e-01, 2.37295032e-01, 7.65312135e-01,
    9.61143553e-01, 4.36596006e-01, 6.77114785e-01, 4.44995224e-01,
    6.83229864e-01, 3.40059429e-01, 6.11794770e-01, 6.90815866e-01,
    9.56425905e-01, 8.06521237e-01, 9.50217426e-01, 9.30451825e-02,
    1.24300353e-01, 6.84115410e-01, 5.58719575e-01, 9.52109814e-01,
    7.08390355e-01, 2.19248887e-02, 7.92785466e-01, 8.06614876e-01,
    2.92044669e-01, 8.83844793e-01, 8.60216200e-01, 9.07594740e-01,
    2.09351406e-01, 9.41360295e-01, 4.94379848e-02, 4.60062832e-01,
    8.38162124e-01, 5.27859628e-01, 3.16174507e-01, 2.18997210e-01,
    5.47150195e-01, 1.86117634e-01, 6.49071097e-01, 6.42057657e-01,
    4.42520112e-01, 7.30912447e-01, 5.05176723e-01, 2.86207590e-02,
    5.84955156e-01, 3.34418684e-01], dtype=float)

def squared_euclidean_distance(vec1, vec2):
    return np.sum((vec1 - vec2) ** 2)

print("Squared Euclidean Distance (5171, Query):", squared_euclidean_distance(dataset_5171, query))
print("Squared Euclidean Distance (9484, Query):", squared_euclidean_distance(dataset_9484, query))

# Squared Euclidean Distance (5171, Query): 4.358781377215945
# Squared Euclidean Distance (9484, Query): 4.358787387166914

[rhdong]

The current conclusion is a random failure caused because the distances are too close. There's still a reminded suspect point as to why the precision looks like a downgrade. As we can see, the Python code shows the differences in the 6th decimal place. But anyway, it does not seem to be a serious issue.