Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add new subdir target #567

Open
wants to merge 3 commits into
base: BANN_save_load_one_file
Choose a base branch
from
Open

add new subdir target #567

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
aa7faac
add new subdir target
MS-Renan Jul 20, 2024
cb3c441
update cmake for diskann_build, update cpp with malloc extension
MS-Renan Aug 6, 2024
06ee93c
remove builddll/
MS-Renan Aug 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 15 additions & 11 deletions src/dll/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,21 @@ if (DISKANN_USE_STATIC_LIB)
../windows_aligned_file_reader.cpp ../distance.cpp ../memory_mapper.cpp ../index.cpp
../in_mem_data_store.cpp ../in_mem_graph_store.cpp ../math_utils.cpp ../disk_utils.cpp ../filter_utils.cpp
../ann_exception.cpp ../natural_number_set.cpp ../natural_number_map.cpp ../scratch.cpp ../index_factory.cpp ../abstract_index.cpp)

# tcmalloc verison
add_library(${PROJECT_NAME}_build STATIC ../abstract_data_store.cpp ../partition.cpp ../pq.cpp ../pq_flash_index.cpp ../logger.cpp ../utils.cpp
../windows_aligned_file_reader.cpp ../distance.cpp ../memory_mapper.cpp ../index.cpp ../../gperftools/src/gperftools/malloc_extension.h
../in_mem_data_store.cpp ../in_mem_graph_store.cpp ../math_utils.cpp ../disk_utils.cpp ../filter_utils.cpp
../ann_exception.cpp ../natural_number_set.cpp ../natural_number_map.cpp ../scratch.cpp ../index_factory.cpp ../abstract_index.cpp)
else()
add_library(${PROJECT_NAME} SHARED dllmain.cpp ../abstract_data_store.cpp ../partition.cpp ../pq.cpp ../pq_flash_index.cpp ../logger.cpp ../utils.cpp
../windows_aligned_file_reader.cpp ../distance.cpp ../memory_mapper.cpp ../index.cpp
../in_mem_data_store.cpp ../in_mem_graph_store.cpp ../math_utils.cpp ../disk_utils.cpp ../filter_utils.cpp
../ann_exception.cpp ../natural_number_set.cpp ../natural_number_map.cpp ../scratch.cpp ../index_factory.cpp ../abstract_index.cpp)

# tcmalloc version
add_library(${PROJECT_NAME}_build SHARED dllmain.cpp ../abstract_data_store.cpp ../partition.cpp ../pq.cpp ../pq_flash_index.cpp ../logger.cpp ../utils.cpp
../windows_aligned_file_reader.cpp ../distance.cpp ../memory_mapper.cpp ../index.cpp
../windows_aligned_file_reader.cpp ../distance.cpp ../memory_mapper.cpp ../index.cpp ../../gperftools/src/gperftools/malloc_extension.h
../in_mem_data_store.cpp ../in_mem_graph_store.cpp ../math_utils.cpp ../disk_utils.cpp ../filter_utils.cpp
../ann_exception.cpp ../natural_number_set.cpp ../natural_number_map.cpp ../scratch.cpp ../index_factory.cpp ../abstract_index.cpp)
endif()
Expand All @@ -21,20 +29,21 @@ set(TARGET_DIR "$<$<CONFIG:Debug>:${CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG}>$<$<CO

if (DISKANN_USE_STATIC_LIB)
target_compile_definitions(${PROJECT_NAME} PRIVATE _NODLL)

target_compile_definitions(${PROJECT_NAME}_build PRIVATE _NODLL DISKANN_BUILD RELEASE_UNUSED_TCMALLOC_MEMORY_AT_CHECKPOINTS)
else()
set(DISKANN_DLL_IMPLIB "${TARGET_DIR}/${PROJECT_NAME}.lib")
target_compile_definitions(${PROJECT_NAME} PRIVATE _USRDLL _WINDLL)

set(DISKANN_DLL_IMPLIB_BUILD "${TARGET_DIR}/${PROJECT_NAME}_build.lib")
target_compile_definitions(${PROJECT_NAME}_build PRIVATE _USRDLL _WINDLL)
target_compile_definitions(${PROJECT_NAME}_build PRIVATE DISKANN_BUILD)
target_compile_definitions(${PROJECT_NAME}_build PRIVATE _USRDLL _WINDLL DISKANN_BUILD RELEASE_UNUSED_TCMALLOC_MEMORY_AT_CHECKPOINTS)
endif()

target_compile_options(${PROJECT_NAME} PRIVATE /GL)
target_include_directories(${PROJECT_NAME} PRIVATE ${DISKANN_MKL_INCLUDE_DIRECTORIES})

target_compile_options(${PROJECT_NAME}_build PRIVATE /GL)
target_include_directories(${PROJECT_NAME}_build PRIVATE ${DISKANN_MKL_INCLUDE_DIRECTORIES})
target_include_directories(${PROJECT_NAME}_build PRIVATE ${DISKANN_MKL_INCLUDE_DIRECTORIES} ../../gperftools/src/)


if (NOT DEFINED DISKANN_USE_STATIC_LIB)
Expand All @@ -44,19 +53,14 @@ endif()

target_link_libraries(${PROJECT_NAME} PRIVATE ${DISKANN_MKL_LINK_LIBRARIES})
target_link_libraries(${PROJECT_NAME} PRIVATE synchronization.lib)

target_link_libraries(${PROJECT_NAME}_build PRIVATE ${DISKANN_MKL_LINK_LIBRARIES})
target_link_libraries(${PROJECT_NAME}_build PRIVATE synchronization.lib)

#This is the crux of the build dll
target_link_libraries(${PROJECT_NAME}_build PUBLIC ${DISKANN_DLL_TCMALLOC_LINK_OPTIONS})
target_link_libraries(${PROJECT_NAME}_build PRIVATE ${TCMALLOC_LINK_LIBRARY})
set_target_properties(${PROJECT_NAME}_build PROPERTIES LINK_FLAGS /INCLUDE:_tcmalloc)



if (DISKANN_DLL_TCMALLOC_LINK_OPTIONS)
target_link_libraries(${PROJECT_NAME} PUBLIC ${DISKANN_DLL_TCMALLOC_LINK_OPTIONS})
endif()

# Copy OpenMP DLL and PDB.
set(RUNTIME_FILES_TO_COPY ${OPENMP_WINDOWS_RUNTIME_FILES} ${TCMALLOC_WINDOWS_RUNTIME_FILES})

Expand Down
4 changes: 4 additions & 0 deletions src/index.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1601,6 +1601,10 @@ template <typename T, typename TagT, typename LabelT> void Index<T, TagT, LabelT
}
}

#if defined(RELEASE_UNUSED_TCMALLOC_MEMORY_AT_CHECKPOINTS) && defined(DISKANN_BUILD)
MallocExtension::instance()->ReleaseFreeMemory();
#endif

if (_nd > 0)
{
diskann::cout << "Starting final cleanup.." << std::flush;
Expand Down
8 changes: 8 additions & 0 deletions src/pq_flash_index.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@
#include "linux_aligned_file_reader.h"
#endif

#if defined(RELEASE_UNUSED_TCMALLOC_MEMORY_AT_CHECKPOINTS) && defined(DISKANN_BUILD)
#include "gperftools/malloc_extension.h"
#endif

#define READ_U64(stream, val) stream.read((char *)&val, sizeof(uint64_t))
#define READ_U32(stream, val) stream.read((char *)&val, sizeof(uint32_t))
#define READ_UNSIGNED(stream, val) stream.read((char *)&val, sizeof(unsigned))
Expand Down Expand Up @@ -92,6 +96,10 @@ template <typename T, typename LabelT> PQFlashIndex<T, LabelT>::~PQFlashIndex()
{
delete[] _pts_to_labels;
}

#if defined(RELEASE_UNUSED_TCMALLOC_MEMORY_AT_CHECKPOINTS) && defined(DISKANN_BUILD)
MallocExtension::instance()->ReleaseFreeMemory();
#endif
}

template <typename T, typename LabelT> inline uint64_t PQFlashIndex<T, LabelT>::get_node_sector(uint64_t node_id)
Expand Down
Loading