Skip to content

Commit 3fc4be7

Browse files
traversctraversc
committed
rcpp mixed env test
1 parent 0ff9917 commit 3fc4be7

10 files changed

Lines changed: 316 additions & 10 deletions

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,3 +39,5 @@ perf.data*
3939
core
4040
core.*
4141
docs/tmp/*
42+
tests/rcpp/.conda-envs/
43+
tests/rcpp/.conda-pkgs/

‎CMakeLists.txt‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,4 +121,20 @@ if(QDATA_BUILD_TESTS)
121121
$<TARGET_FILE:qdata_roundtrip_file>
122122
$<TARGET_FILE:qdata_roundtrip_memory>
123123
)
124+
find_program(CONDA_EXECUTABLE conda QUIET)
125+
if(CONDA_EXECUTABLE)
126+
set(QDATA_RCPP_TEST_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/tests/rcpp/run_flow_compat_test.sh)
127+
add_test(
128+
NAME qdata_rcppparallel_flow_compat
129+
COMMAND bash
130+
${QDATA_RCPP_TEST_SCRIPT}
131+
${CONDA_EXECUTABLE}
132+
)
133+
set_tests_properties(
134+
PROPERTIES
135+
TIMEOUT 1800
136+
)
137+
else()
138+
message(STATUS "conda not found; skipping RcppParallel conda-backed tests")
139+
endif()
124140
endif()

‎Makefile‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ QS_EXTENDED_TESTS ?= 1
77
ROWS ?=
88
REPS ?=
99
LICENSE_DIR ?= LICENSES
10+
RCPP_TEST_ENVS_DIR ?= tests/rcpp/.conda-envs
1011

1112
BENCH_ARGS :=
1213
ifneq ($(strip $(ROWS)),)
@@ -16,7 +17,7 @@ ifneq ($(strip $(REPS)),)
1617
BENCH_ARGS += $(REPS)
1718
endif
1819

19-
.PHONY: all configure example bench benchmark benchmark-build test test-extended get-license-files clean distclean
20+
.PHONY: all configure example bench benchmark benchmark-build test test-extended test-rcpp-parallel get-license-files clean distclean
2021

2122
all: configure
2223
$(CMAKE) --build $(BUILD_DIR)
@@ -42,6 +43,10 @@ test-extended: configure
4243
$(CMAKE) --build $(BUILD_DIR)
4344
QS_EXTENDED_TESTS=$(QS_EXTENDED_TESTS) $(CTEST) --test-dir $(BUILD_DIR) $(CTEST_FLAGS)
4445

46+
test-rcpp-parallel: configure
47+
$(CTEST) --test-dir $(BUILD_DIR) $(CTEST_FLAGS) -R '^qdata_rcppparallel_flow_compat$$'
48+
49+
4550
get-license-files:
4651
@mkdir -p $(LICENSE_DIR)
4752
curl -fsSL https://www.gnu.org/licenses/gpl-3.0.txt -o LICENSE
@@ -50,6 +55,10 @@ get-license-files:
5055

5156
clean:
5257
@if [ -d "$(BUILD_DIR)" ]; then $(CMAKE) --build $(BUILD_DIR) --target clean; fi
58+
@if [ -d "$(RCPP_TEST_ENVS_DIR)" ]; then $(CMAKE) -E rm -rf "$(RCPP_TEST_ENVS_DIR)"; fi
59+
@if [ -d "tests/rcpp/.conda-pkgs" ]; then $(CMAKE) -E rm -rf "tests/rcpp/.conda-pkgs"; fi
5360

5461
distclean:
5562
$(CMAKE) -E rm -rf $(BUILD_DIR)
63+
$(CMAKE) -E rm -rf $(RCPP_TEST_ENVS_DIR)
64+
$(CMAKE) -E rm -rf tests/rcpp/.conda-pkgs

‎include/io/tbb_flow_compat.h‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,24 @@
44
#include <utility>
55
#include <tbb/flow_graph.h>
66

7-
// TBB 2019 exposes only <tbb/flow_graph.h> and uses source_node/decrement.
8-
// oneTBB ships its canonical flow-graph API under <oneapi/tbb/flow_graph.h>
9-
// and uses input_node/decrementer(). Probe that header layout instead of
10-
// version macros: some wrapper headers do not surface TBB_INTERFACE_VERSION,
11-
// and mixing version headers from different packages can mis-detect the actual
12-
// flow-graph API in use. Callers can override this probe if they know better.
7+
// TBB 2019 uses source_node/decrement and surfaces TBB_INTERFACE_VERSION via
8+
// tbb_stddef.h. oneTBB uses input_node/decrementer(), but <tbb/flow_graph.h>
9+
// wrappers do not always surface the version macro. Prefer the interface
10+
// version when the active flow-graph include exposes it, otherwise fall back
11+
// to the old-header guard instead of probing unrelated oneapi/ headers that
12+
// may come from a different package on the include path. Callers can override
13+
// this probe if they know better.
1314
#ifndef QIO_TBB_FLOW_USES_INPUT_NODE
14-
#if defined(__has_include)
15-
#if __has_include(<oneapi/tbb/flow_graph.h>)
15+
#if defined(TBB_INTERFACE_VERSION)
16+
#if TBB_INTERFACE_VERSION >= 12000
1617
#define QIO_TBB_FLOW_USES_INPUT_NODE 1
1718
#else
1819
#define QIO_TBB_FLOW_USES_INPUT_NODE 0
1920
#endif
20-
#else
21+
#elif defined(__TBB_tbb_stddef_H)
2122
#define QIO_TBB_FLOW_USES_INPUT_NODE 0
23+
#else
24+
#define QIO_TBB_FLOW_USES_INPUT_NODE 1
2225
#endif
2326
#endif
2427

‎tests/rcpp/environment-conda.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
name: qdata-rcpp-conda
2+
channels:
3+
- conda-forge
4+
dependencies:
5+
- r-base
6+
- r-rcpp
7+
- r-rcppparallel

‎tests/rcpp/environment-cran.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
name: qdata-rcpp-cran
2+
channels:
3+
- conda-forge
4+
dependencies:
5+
- r-base
6+
- r-rcpp

‎tests/rcpp/environment-mixed.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
name: qdata-rcpp-mixed
2+
channels:
3+
- conda-forge
4+
dependencies:
5+
- r-base
6+
- r-rcpp
7+
- tbb-devel

‎tests/rcpp/run_flow_compat_test.sh‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
if [ "$#" -ne 1 ]; then
5+
echo "usage: $0 <conda-executable>" >&2
6+
exit 2
7+
fi
8+
9+
conda_executable=$1
10+
11+
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
12+
envs_dir="$script_dir/.conda-envs"
13+
pkgs_dir="$script_dir/.conda-pkgs"
14+
setup_tag="qdata-flow-compat-v1"
15+
16+
hash_file() {
17+
if command -v sha256sum >/dev/null 2>&1; then
18+
sha256sum "$1" | awk '{print $1}'
19+
elif command -v shasum >/dev/null 2>&1; then
20+
shasum -a 256 "$1" | awk '{print $1}'
21+
else
22+
echo "missing sha256 tool" >&2
23+
exit 2
24+
fi
25+
}
26+
27+
has_conda_meta() {
28+
local env_prefix=$1
29+
local package_name=$2
30+
31+
compgen -G "$env_prefix/conda-meta/$package_name-*.json" >/dev/null
32+
}
33+
34+
capture_versions() {
35+
local env_prefix=$1
36+
37+
CONDA_PKGS_DIRS="$pkgs_dir" "$conda_executable" run --prefix "$env_prefix" \
38+
Rscript -e 'pkgs <- c("Rcpp", "RcppParallel"); stopifnot(all(vapply(pkgs, requireNamespace, logical(1), quietly = TRUE))); cat(as.character(packageVersion("Rcpp")), "\n", as.character(packageVersion("RcppParallel")), "\n", sep = "")'
39+
}
40+
41+
write_stamp() {
42+
local env_kind=$1
43+
local source_kind=$2
44+
local env_prefix="$envs_dir/$env_kind"
45+
local env_file="$script_dir/environment-$env_kind.yml"
46+
local stamp_file="$env_prefix/.qdata-flow-compat-stamp"
47+
local versions
48+
local rcpp_version
49+
local rcppparallel_version
50+
51+
versions=$(capture_versions "$env_prefix")
52+
rcpp_version=$(printf '%s\n' "$versions" | sed -n '1p')
53+
rcppparallel_version=$(printf '%s\n' "$versions" | sed -n '2p')
54+
55+
cat > "$stamp_file" <<EOF
56+
STAMP_ENV_HASH=$(hash_file "$env_file")
57+
STAMP_SETUP_TAG=$setup_tag
58+
STAMP_SOURCE_KIND=$source_kind
59+
STAMP_RCPP_VERSION=$rcpp_version
60+
STAMP_RCPPPARALLEL_VERSION=$rcppparallel_version
61+
EOF
62+
}
63+
64+
env_is_ready() {
65+
local env_kind=$1
66+
local source_kind=$2
67+
local env_prefix="$envs_dir/$env_kind"
68+
local env_file="$script_dir/environment-$env_kind.yml"
69+
local stamp_file="$env_prefix/.qdata-flow-compat-stamp"
70+
local versions
71+
local rcpp_version
72+
local rcppparallel_version
73+
74+
if [ ! -x "$env_prefix/bin/Rscript" ] || [ ! -f "$stamp_file" ]; then
75+
return 1
76+
fi
77+
78+
STAMP_ENV_HASH=
79+
STAMP_SETUP_TAG=
80+
STAMP_SOURCE_KIND=
81+
STAMP_RCPP_VERSION=
82+
STAMP_RCPPPARALLEL_VERSION=
83+
# shellcheck disable=SC1090
84+
. "$stamp_file"
85+
86+
if [ "$STAMP_ENV_HASH" != "$(hash_file "$env_file")" ]; then
87+
return 1
88+
fi
89+
if [ "$STAMP_SETUP_TAG" != "$setup_tag" ]; then
90+
return 1
91+
fi
92+
if [ "$STAMP_SOURCE_KIND" != "$source_kind" ]; then
93+
return 1
94+
fi
95+
96+
case "$source_kind" in
97+
conda)
98+
has_conda_meta "$env_prefix" r-rcppparallel || return 1
99+
;;
100+
cran)
101+
if has_conda_meta "$env_prefix" r-rcppparallel; then
102+
return 1
103+
fi
104+
;;
105+
*)
106+
echo "unexpected source kind: $source_kind" >&2
107+
exit 2
108+
;;
109+
esac
110+
111+
if [ "$env_kind" = mixed ]; then
112+
has_conda_meta "$env_prefix" tbb-devel || return 1
113+
fi
114+
115+
if ! versions=$(capture_versions "$env_prefix"); then
116+
return 1
117+
fi
118+
119+
rcpp_version=$(printf '%s\n' "$versions" | sed -n '1p')
120+
rcppparallel_version=$(printf '%s\n' "$versions" | sed -n '2p')
121+
122+
[ "$rcpp_version" = "$STAMP_RCPP_VERSION" ] || return 1
123+
[ "$rcppparallel_version" = "$STAMP_RCPPPARALLEL_VERSION" ] || return 1
124+
}
125+
126+
rebuild_env() {
127+
local env_kind=$1
128+
local env_file="$script_dir/environment-$env_kind.yml"
129+
local env_prefix="$envs_dir/$env_kind"
130+
local log_file
131+
132+
rm -rf "$env_prefix"
133+
mkdir -p "$envs_dir" "$pkgs_dir"
134+
log_file=$(mktemp)
135+
136+
if CONDA_PKGS_DIRS="$pkgs_dir" "$conda_executable" env create --prefix "$env_prefix" --file "$env_file" --yes 2>&1 | tee "$log_file"; then
137+
rm -f "$log_file"
138+
return
139+
fi
140+
141+
if grep -Eiq 'SafetyError|corrupt|incorrect size' "$log_file"; then
142+
echo "Detected corrupt local conda package cache, clearing $pkgs_dir and retrying once"
143+
rm -rf "$pkgs_dir" "$env_prefix"
144+
mkdir -p "$pkgs_dir"
145+
CONDA_PKGS_DIRS="$pkgs_dir" "$conda_executable" env create --prefix "$env_prefix" --file "$env_file" --yes
146+
rm -f "$log_file"
147+
return
148+
fi
149+
150+
rm -f "$log_file"
151+
return 1
152+
}
153+
154+
install_cran_rcppparallel() {
155+
local env_kind=$1
156+
local env_prefix="$envs_dir/$env_kind"
157+
158+
CONDA_PKGS_DIRS="$pkgs_dir" "$conda_executable" run --prefix "$env_prefix" Rscript -e 'options(repos = c(CRAN = "https://cloud.r-project.org")); install.packages("RcppParallel", type = "source")'
159+
}
160+
161+
ensure_env() {
162+
local env_kind=$1
163+
local source_kind=$2
164+
165+
if env_is_ready "$env_kind" "$source_kind"; then
166+
echo "Reusing $env_kind env"
167+
return
168+
fi
169+
170+
echo "Rebuilding $env_kind env"
171+
rebuild_env "$env_kind"
172+
if [ "$source_kind" = cran ]; then
173+
install_cran_rcppparallel "$env_kind"
174+
fi
175+
write_stamp "$env_kind" "$source_kind"
176+
}
177+
178+
run_probe() {
179+
local env_kind=$1
180+
local env_prefix="$envs_dir/$env_kind"
181+
182+
CONDA_PKGS_DIRS="$pkgs_dir" "$conda_executable" run --prefix "$env_prefix" Rscript "$script_dir/tbb_flow_compat.R" "$script_dir/tbb_flow_compat_probe.cpp"
183+
}
184+
185+
echo "Testing RcppParallel from conda (oneTBB)"
186+
ensure_env conda conda
187+
run_probe conda
188+
189+
echo "Testing RcppParallel from CRAN (TBB 2019)"
190+
ensure_env cran cran
191+
run_probe cran
192+
193+
echo "Testing RcppParallel from CRAN with system tbb-devel (mixed header test)"
194+
ensure_env mixed cran
195+
run_probe mixed

‎tests/rcpp/tbb_flow_compat.R‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
args <- commandArgs(TRUE)
2+
source_file <- normalizePath(args[[1]], winslash = "/", mustWork = TRUE)
3+
4+
Rcpp::sourceCpp(
5+
file = source_file,
6+
rebuild = TRUE,
7+
showOutput = TRUE
8+
)
9+
10+
result <- qdata_tbb_flow_compat_probe()
11+
12+
stopifnot(is.list(result))
13+
stopifnot(identical(as.integer(result$produced), 3L))
14+
stopifnot(identical(as.integer(result$consumed), 3L))
15+
stopifnot(identical(as.integer(result$sum), 3L))

‎tests/rcpp/tbb_flow_compat_probe.cpp‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// [[Rcpp::depends(RcppParallel)]]
2+
// [[Rcpp::plugins(cpp17)]]
3+
#include <Rcpp.h>
4+
5+
#include "../../include/io/tbb_flow_compat.h"
6+
7+
// [[Rcpp::export]]
8+
Rcpp::List qdata_tbb_flow_compat_probe() {
9+
tbb::flow::graph graph;
10+
int next_value = 0;
11+
int produced = 0;
12+
int consumed = 0;
13+
int sum = 0;
14+
15+
qio::tbb_compat::source_node<int> source(graph,
16+
[&next_value, &produced](int & value) {
17+
if(next_value >= 3) {
18+
return false;
19+
}
20+
value = next_value++;
21+
++produced;
22+
return true;
23+
});
24+
25+
tbb::flow::limiter_node<int> limiter(graph, 1);
26+
tbb::flow::function_node<int, tbb::flow::continue_msg> sink(graph, tbb::flow::serial,
27+
[&consumed, &sum](int value) {
28+
++consumed;
29+
sum += value;
30+
return tbb::flow::continue_msg{};
31+
});
32+
33+
tbb::flow::make_edge(source, limiter);
34+
tbb::flow::make_edge(limiter, sink);
35+
tbb::flow::make_edge(sink, qio::tbb_compat::decrementer(limiter));
36+
37+
source.activate();
38+
graph.wait_for_all();
39+
40+
return Rcpp::List::create(
41+
Rcpp::Named("uses_input_node") = static_cast<bool>(QIO_TBB_FLOW_USES_INPUT_NODE),
42+
Rcpp::Named("produced") = produced,
43+
Rcpp::Named("consumed") = consumed,
44+
Rcpp::Named("sum") = sum
45+
);
46+
}

0 commit comments

Comments
 (0)