Hide visibility of non public symbols (rapidsai#15982)

Converts cudf over to a system of explicit markup of what symbols should be used by consumers. This is done by compiling with `-fvisibility=hidden` and explicit markup via `CUDF_EXPORT` of components we want usable.

Due to issues with tests a portion of `include/` detail functions had to be marked as public API.

More concernning are that the tests leverage functions from `cpp/` that are never part of the installed headers. That set of files can be found at rapidsai@16b3656 and we should discuss how we should restructure cudf to remove these.

Authors:
  - Robert Maynard (https://github.com/robertmaynard)
  - Bradley Dice (https://github.com/bdice)

Approvers:
  - Bradley Dice (https://github.com/bdice)
  - Nghia Truong (https://github.com/ttnghia)

URL: rapidsai#15982

Author: robertmaynard

cpp/CMakeLists.txt

@@ -711,8 +711,10 @@ set_target_properties(
              CXX_STANDARD_REQUIRED ON
              # For std:: support of __int128_t. Can be removed once using cuda::std
              CXX_EXTENSIONS ON
+             CXX_VISIBILITY_PRESET hidden
              CUDA_STANDARD 17
              CUDA_STANDARD_REQUIRED ON
+             CUDA_VISIBILITY_PRESET hidden
              POSITION_INDEPENDENT_CODE ON
              INTERFACE_POSITION_INDEPENDENT_CODE ON
 )
@@ -887,8 +889,10 @@ if(CUDF_BUILD_TESTUTIL)
                # set target compile options
                CXX_STANDARD 17
                CXX_STANDARD_REQUIRED ON
+               CXX_VISIBILITY_PRESET hidden
                CUDA_STANDARD 17
                CUDA_STANDARD_REQUIRED ON
+               CUDA_VISIBILITY_PRESET hidden
                POSITION_INDEPENDENT_CODE ON
                INTERFACE_POSITION_INDEPENDENT_CODE ON
   )

cpp/cmake/thirdparty/patches/cccl_override.json

@@ -3,6 +3,11 @@
   "packages" : {
     "CCCL" : {
       "patches" : [
+        {
+          "file" : "${current_json_dir}/cccl_symbol_visibility.diff",
+          "issue" : "Correct symbol visibility issues in libcudacxx [https://github.com/NVIDIA/cccl/pull/1832/]",
+          "fixed_in" : "2.6"
+        },
         {
           "file" : "${current_json_dir}/thrust_disable_64bit_dispatching.diff",
           "issue" : "Remove 64bit dispatching as not needed by libcudf and results in compiling twice as many kernels [https://github.com/rapidsai/cudf/pull/11437]",

cpp/cmake/thirdparty/patches/cccl_symbol_visibility.diff

@@ -0,0 +1,27 @@
+diff --git a/libcudacxx/include/cuda/std/detail/libcxx/include/__config b/libcudacxx/include/cuda/std/detail/libcxx/include/__config
+index e7c62c031b..5db861853a 100644
+--- a/libcudacxx/include/cuda/std/detail/libcxx/include/__config
++++ b/libcudacxx/include/cuda/std/detail/libcxx/include/__config
+@@ -1049,7 +1049,6 @@ typedef __char32_t char32_t;
+ #      define _LIBCUDACXX_EXPORTED_FROM_ABI __declspec(dllimport)
+ #    endif
+ 
+-#    define _LIBCUDACXX_TYPE_VIS      _LIBCUDACXX_DLL_VIS
+ #    define _LIBCUDACXX_FUNC_VIS      _LIBCUDACXX_DLL_VIS
+ #    define _LIBCUDACXX_EXCEPTION_ABI _LIBCUDACXX_DLL_VIS
+ #    define _LIBCUDACXX_HIDDEN
+@@ -1448,14 +1447,6 @@ __sanitizer_annotate_contiguous_container(const void*, const void*, const void*,
+ #    define _LIBCUDACXX_WEAK __attribute__((__weak__))
+ #  endif
+ 
+-// Redefine some macros for internal use
+-#  if defined(__cuda_std__)
+-#    undef _LIBCUDACXX_FUNC_VIS
+-#    define _LIBCUDACXX_FUNC_VIS _LIBCUDACXX_INLINE_VISIBILITY
+-#    undef _LIBCUDACXX_TYPE_VIS
+-#    define _LIBCUDACXX_TYPE_VIS
+-#  endif // __cuda_std__
+-
+ // Thread API
+ #  ifndef _LIBCUDACXX_HAS_THREAD_API_EXTERNAL
+ #    if defined(_CCCL_COMPILER_NVRTC) || defined(__EMSCRIPTEN__)

cpp/doxygen/developer_guide/DEVELOPER_GUIDE.md

@@ -52,15 +52,36 @@ header file in `cudf/cpp/include/cudf/`. For example, `cudf/cpp/include/cudf/cop
 contains the APIs for functions related to copying from one column to another. Note the `.hpp`
 file extension used to indicate a C++ header file.
 
-Header files should use the `#pragma once` include guard.
+External/public libcudf C++ API header files need to mark all symbols inside of them with `CUDF_EXPORT`.
+This is done by placing the macro on the `namespace cudf` as seen below. Markup on namespace
+require them not to be nested, so the `cudf` namespace must be kept by itself.
+
+```c++
+
+#pragma once
+
+namespace CUDF_EXPORT cudf {
+namespace lists {
+
+...
+
+
+} // namespace lists
+} // namespace CUDF_EXPORT cudf
+
+```
+
 
 The naming of external API headers should be consistent with the name of the folder that contains
 the source files that implement the API. For example, the implementation of the APIs found in
 `cudf/cpp/include/cudf/copying.hpp` are located in `cudf/src/copying`. Likewise, the unit tests for
 the APIs reside in `cudf/tests/copying/`.
 
-Internal API headers containing `detail` namespace definitions that are used across translation
-units inside libcudf should be placed in `include/cudf/detail`.
+Internal API headers containing `detail` namespace definitions that are either used across translation
+units inside libcudf should be placed in `include/cudf/detail`. Just like the public C++ API headers, any
+internal C++ API header requires `CUDF_EXPORT` markup on the `cudf` namespace so that the functions can be tested.
+
+All headers in cudf should use `#pragma once` for include guards.
 
 ## File extensions
 

cpp/doxygen/developer_guide/DOCUMENTATION.md

@@ -363,7 +363,7 @@ Here is an example of a doxygen description comment for a namespace declaration.
      *
      * This is the top-level namespace which contains all cuDF functions and types.
      */
-    namespace cudf {
+    namespace CUDF_EXPORT cudf {
 
 A description comment should be included only once for each unique namespace declaration.
 Otherwise, if more than one description is found, doxygen aggregates the descriptions in an arbitrary order in the output pages.
@@ -385,7 +385,7 @@ The existing groups have been carefully structured and named, so new groups shou
 
 When creating a new API, specify its group using the [\@ingroup](https://www.doxygen.nl/manual/commands.html#cmdingroup) tag and the group reference id from the [doxygen_groups.h](../include/doxygen_groups.h) file.
 
-    namespace cudf {
+    namespace CUDF_EXPORT cudf {
 
     /**
      * @brief ...
@@ -401,7 +401,7 @@ When creating a new API, specify its group using the [\@ingroup](https://www.dox
 
 You can also use the \@addtogroup with a `@{ ... @}` pair to automatically include doxygen comment blocks as part of a group.
 
-    namespace cudf {
+    namespace CUDF_EXPORT cudf {
     /**
      * @addtogroup transformation_fill
      * @{

cpp/include/cudf/aggregation.hpp

@@ -17,6 +17,7 @@
 #pragma once
 
 #include <cudf/types.hpp>
+#include <cudf/utilities/export.hpp>
 
 #include <functional>
 #include <memory>
@@ -31,7 +32,7 @@
  * individual function documentation to see what aggregations are supported.
  */
 
-namespace cudf {
+namespace CUDF_EXPORT cudf {
 /**
  * @addtogroup aggregation_factories
  * @{
@@ -770,4 +771,4 @@ template <typename Base>
 std::unique_ptr<Base> make_merge_tdigest_aggregation(int max_centroids = 1000);
 
 /** @} */  // end of group
-}  // namespace cudf
+}  // namespace CUDF_EXPORT cudf

cpp/include/cudf/ast/detail/expression_parser.hpp

@@ -29,9 +29,8 @@
 #include <numeric>
 #include <optional>
 
-namespace cudf {
-namespace ast {
-namespace detail {
+namespace CUDF_EXPORT cudf {
+namespace ast::detail {
 
 /**
  * @brief Node data reference types.
@@ -328,8 +327,6 @@ class expression_parser {
   std::vector<generic_scalar_device_view> _literals;
 };
 
-}  // namespace detail
+}  // namespace ast::detail
 
-}  // namespace ast
-
-}  // namespace cudf
+}  // namespace CUDF_EXPORT cudf

cpp/include/cudf/ast/detail/expression_transformer.hpp

@@ -1,6 +1,6 @@
 
 /*
- * Copyright (c) 2023, NVIDIA CORPORATION.
+ * Copyright (c) 2023-2024, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,7 +18,8 @@
 
 #include <cudf/ast/expressions.hpp>
 
-namespace cudf::ast::detail {
+namespace CUDF_EXPORT cudf {
+namespace ast::detail {
 /**
  * @brief Base "visitor" pattern class with the `expression` class for expression transformer.
  *
@@ -61,4 +62,7 @@ class expression_transformer {
 
   virtual ~expression_transformer() {}
 };
-}  // namespace cudf::ast::detail
+
+}  // namespace ast::detail
+
+}  // namespace CUDF_EXPORT cudf

cpp/include/cudf/ast/detail/operators.hpp

@@ -29,7 +29,7 @@
 #include <utility>
 #include <vector>
 
-namespace cudf {
+namespace CUDF_EXPORT cudf {
 
 namespace ast {
 
@@ -1233,4 +1233,4 @@ CUDF_HOST_DEVICE inline cudf::size_type ast_operator_arity(ast_operator op)
 
 }  // namespace ast
 
-}  // namespace cudf
+}  // namespace CUDF_EXPORT cudf

cpp/include/cudf/ast/expressions.hpp

@@ -23,7 +23,7 @@
 
 #include <cstdint>
 
-namespace cudf {
+namespace CUDF_EXPORT cudf {
 namespace ast {
 /**
  * @addtogroup expressions
@@ -555,4 +555,4 @@ class column_name_reference : public expression {
 /** @} */  // end of group
 }  // namespace ast
 
-}  // namespace cudf
+}  // namespace CUDF_EXPORT cudf

cpp/include/cudf/binaryop.hpp

@@ -18,13 +18,14 @@
 
 #include <cudf/column/column.hpp>
 #include <cudf/scalar/scalar.hpp>
+#include <cudf/utilities/export.hpp>
 
 #include <rmm/mr/device/per_device_resource.hpp>
 #include <rmm/resource_ref.hpp>
 
 #include <memory>
 
-namespace cudf {
+namespace CUDF_EXPORT cudf {
 
 /**
  * @addtogroup transformation_binaryops
@@ -316,8 +317,13 @@ std::pair<rmm::device_buffer, size_type> scalar_col_valid_mask_and(
   rmm::cuda_stream_view stream      = cudf::get_default_stream(),
   rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource());
 
-namespace compiled {
-namespace detail {
+}  // namespace binops
+
+/** @} */  // end of group
+}  // namespace CUDF_EXPORT cudf
+
+namespace CUDF_EXPORT cudf {
+namespace binops::compiled::detail {
 
 /**
  * @brief struct binary operation using `NaN` aware sorting physical element comparators
@@ -337,9 +343,5 @@ void apply_sorting_struct_binary_op(mutable_column_view& out,
                                     bool is_rhs_scalar,
                                     binary_operator op,
                                     rmm::cuda_stream_view stream);
-}  // namespace detail
-}  // namespace compiled
-}  // namespace binops
-
-/** @} */  // end of group
-}  // namespace cudf
+}  // namespace binops::compiled::detail
+}  // namespace CUDF_EXPORT cudf

cpp/include/cudf/column/column.hpp

@@ -36,7 +36,7 @@
  * @brief Class definition for cudf::column
  */
 
-namespace cudf {
+namespace CUDF_EXPORT cudf {
 
 /**
  * @brief A container of nullable device data as a column of elements.
@@ -332,4 +332,4 @@ class column {
 };
 
 /** @} */  // end of group
-}  // namespace cudf
+}  // namespace CUDF_EXPORT cudf

cpp/include/cudf/column/column_device_view.cuh

@@ -44,7 +44,7 @@
  * @brief Column device view class definitions
  */
 
-namespace cudf {
+namespace CUDF_EXPORT cudf {
 
 /**
  * @brief Indicates the presence of nulls at compile-time or runtime.
@@ -1527,4 +1527,4 @@ ColumnDeviceView* child_columns_to_device_array(ColumnViewIterator child_begin,
 }
 
 }  // namespace detail
-}  // namespace cudf
+}  // namespace CUDF_EXPORT cudf

cpp/include/cudf/column/column_factories.hpp

@@ -27,7 +27,7 @@
 
 #include <thrust/pair.h>
 
-namespace cudf {
+namespace CUDF_EXPORT cudf {
 /**
  * @addtogroup column_factories
  * @{
@@ -571,4 +571,4 @@ std::unique_ptr<column> make_dictionary_from_scalar(
   rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource());
 
 /** @} */  // end of group
-}  // namespace cudf
+}  // namespace CUDF_EXPORT cudf

cpp/include/cudf/column/column_view.hpp

@@ -31,8 +31,7 @@
  * @file column_view.hpp
  * @brief column view class definitions
  */
-
-namespace cudf {
+namespace CUDF_EXPORT cudf {
 namespace detail {
 /**
  * @brief A non-owning, immutable view of device data as a column of elements,
@@ -296,6 +295,7 @@ class column_view_base {
                    size_type null_count,
                    size_type offset = 0);
 };
+
 }  // namespace detail
 
 /**
@@ -797,5 +797,6 @@ std::size_t shallow_hash(column_view const& input);
  * @return If `lhs` and `rhs` have equivalent shallow state
  */
 bool is_shallow_equivalent(column_view const& lhs, column_view const& rhs);
+
 }  // namespace detail
-}  // namespace cudf
+}  // namespace CUDF_EXPORT cudf

cpp/include/cudf/concatenate.hpp

@@ -18,14 +18,15 @@
 #include <cudf/column/column_view.hpp>
 #include <cudf/table/table_view.hpp>
 #include <cudf/utilities/default_stream.hpp>
+#include <cudf/utilities/export.hpp>
 #include <cudf/utilities/span.hpp>
 
 #include <rmm/mr/device/per_device_resource.hpp>
 #include <rmm/resource_ref.hpp>
 
 #include <memory>
 
-namespace cudf {
+namespace CUDF_EXPORT cudf {
 /**
  * @addtogroup copy_concatenate
  * @{
@@ -97,4 +98,4 @@ std::unique_ptr<table> concatenate(
   rmm::device_async_resource_ref mr = rmm::mr::get_current_device_resource());
 
 /** @} */  // end of group
-}  // namespace cudf
+}  // namespace CUDF_EXPORT cudf