Skip to content

Fix testcases #439

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MaggieQi wants to merge 29 commits into
base: main
Choose a base branch
from
Open

Fix testcases #439

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
351f03c
update to vc143
Jun 4, 2025
1af89ef
merge
Nov 20, 2025
e762fd5
move mergepostings to refineindex
Nov 22, 2025
57bf5ed
fix errorcode
Nov 25, 2025
f8ea350
Add checksum verification in merge
Dec 3, 2025
574412e
add configuable merge during search and benchmark test
Dec 4, 2025
b79b98a
fix linux localtime_s to localtime_r
Dec 4, 2025
a26f2ed
fix the benchmark to avoid create workspace after reload during search
Dec 11, 2025
da41c2b
add output file to the benchmark
Dec 11, 2025
c1036bc
fix the BatchNum
Dec 11, 2025
20fa847
fix benchmark to include multiple batch perf
Dec 12, 2025
fa5fb29
write the result in time
Dec 12, 2025
bcafeb7
let benchmark fit for larger dataset
Dec 12, 2025
3107cf6
ensure metadata size
Dec 12, 2025
b87551c
add build time
Dec 12, 2025
40a89d6
add rebuild and resume configuration
Dec 12, 2025
b78aa90
change printstep
Dec 12, 2025
d29ce73
fix print step
Dec 13, 2025
c6e0dfc
add more append threads
Dec 15, 2025
e06a423
add more log and change Multiget to use buffer
Dec 16, 2025
2e28ed2
clean some logs
Dec 16, 2025
1e08a00
ensure workspace posting size.
Dec 17, 2025
d3f2a76
add Get to PageBuffer API
Dec 17, 2025
1fe1bb3
fix cache issue
Dec 19, 2025
89cac4b
fix unsafe_delete
Dec 30, 2025
2bc00ca
fix unsafe delete
Dec 30, 2025
130dcfb
avoid dataset incblock leak
Dec 31, 2025
f771a7c
Add split VID check
Jan 2, 2026
6714e5a
fix dataset
Jan 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions AnnService/Aggregator.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,13 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
Expand Down
4 changes: 2 additions & 2 deletions AnnService/BalancedDataPartition.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,13 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
Expand Down
4 changes: 2 additions & 2 deletions AnnService/Client.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,13 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
Expand Down
4 changes: 2 additions & 2 deletions AnnService/IndexBuilder.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,13 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
Expand Down
4 changes: 2 additions & 2 deletions AnnService/IndexSearcher.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,13 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
Expand Down
4 changes: 2 additions & 2 deletions AnnService/SSDServing.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,13 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
Expand Down
4 changes: 2 additions & 2 deletions AnnService/Server.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,13 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
Expand Down
4 changes: 2 additions & 2 deletions AnnService/SocketLib.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,13 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<PlatformToolset>v142</PlatformToolset>
<PlatformToolset>v143</PlatformToolset>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
Expand Down
24 changes: 24 additions & 0 deletions AnnService/inc/Core/Common/CommonUtils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,30 @@ namespace SPTAG
}
}
}

static void PrintPostingDiff(std::string &p1, std::string &p2, const char *pos)
{
if (p1.size() != p2.size())
{
SPTAGLIB_LOG(Helper::LogLevel::LL_Error,
"Merge %s: p1 and p2 have different sizes: before=%u after=%u\n", pos, p1.size(),
p2.size());
return;
}
std::string diff = "";
for (size_t i = 0; i < p1.size(); i++)
{
if (p1[i] != p2[i])
{
diff += "[" + std::to_string(i) + "]:" + std::to_string(int(p1[i])) + "^" +
std::to_string(int(p2[i])) + " ";
}
}
if (diff.size() != 0)
{
SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "%s: %s\n", pos, diff.substr(0, 1000).c_str());
}
}
};
}
}
Expand Down
18 changes: 10 additions & 8 deletions AnnService/inc/Core/Common/Dataset.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,11 +260,17 @@ namespace SPTAG
{
if (data != nullptr) {
if (ownData) ALIGN_FREE(data);
for (char* ptr : *incBlocks) ALIGN_FREE(ptr);
incBlocks->clear();
}

rows = rows_;

if (incBlocks != nullptr)
{
for (char *ptr : *incBlocks)
ALIGN_FREE(ptr);
incBlocks->clear();
}
incRows = 0;

if (rowEnd_ >= colStart_) cols = rowEnd_;
else cols = cols_ * sizeof(T);
data = (char*)data_;
Expand Down Expand Up @@ -390,11 +396,7 @@ namespace SPTAG
IOBINARY(pInput, ReadBinary, sizeof(SizeType), (char*)&(r));
IOBINARY(pInput, ReadBinary, sizeof(DimensionType), (char*)(&c));

if (data == nullptr) Initialize(r, c, blockSize, capacity);
else if (r > rows + incRows) {
SPTAGLIB_LOG(Helper::LogLevel::LL_Warning, "Read more data (%d, %d) than before (%d, %d)!\n", r, c, rows + incRows, mycols);
if(AddBatch(r - rows - incRows) != ErrorCode::Success) return ErrorCode::MemoryOverFlow;
}
if (data == nullptr || r != rows + incRows) Initialize(r, c, blockSize, capacity);

for (SizeType i = 0; i < r; i++) {
IOBINARY(pInput, ReadBinary, sizeof(T) * mycols, (char*)At(i));
Expand Down
4 changes: 2 additions & 2 deletions AnnService/inc/Core/MetadataSet.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ class MemMetadataSet : public MetadataSet
std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize);

MemMetadataSet(const std::string& p_metafile, const std::string& p_metaindexfile,
std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize);
std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize, int start = 0, int count = -1);

MemMetadataSet(std::shared_ptr<Helper::DiskIO> p_metain, std::shared_ptr<Helper::DiskIO> p_metaindexin,
std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize);
Expand All @@ -118,7 +118,7 @@ class MemMetadataSet : public MetadataSet

private:
ErrorCode Init(std::shared_ptr<Helper::DiskIO> p_metain, std::shared_ptr<Helper::DiskIO> p_metaindexin,
std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize);
std::uint64_t p_blockSize, std::uint64_t p_capacity, std::uint64_t p_metaSize, int start = 0, int count = -1);

std::shared_ptr<void> m_lock;

Expand Down
Loading
Loading