How to use google benchmark with cmake and different compilers
Google benchmark is a great tool for measuring performance of algorithms.
But it may become tricky using it with cmake projects, different compilers and not beeing installed globally on your system.
Google benchmark
First of all you should organize your project folders.
I use a project folder and clone google benchmark as gbenchmark directory into it.
To allow multiple compilers, I name the build folders by myself and will use build-gcc13 for gcc 13.2 and build-clang for Apple clang 15.0.
gcc
Lets start with gcc:
~/project $ git clone https://github.com/google/benchmark.git ./gbenchmark
~/project $ cd gbenchmark
~/project/gbenchmark $ cmake -E make_directory "build-gcc13"
To tell cmake which compiler to use (if you have more than one installed), just set CMAKE_*_COMPILER when creating the project. This can be either a fully qualified path or - if available - a link to the executable:
~/project/gbenchmark $ cmake -E chdir "build-gcc13" cmake -DBENCHMARK_DOWNLOAD_DEPENDENCIES=on
-DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=gcc-13 -DCMAKE_CXX_COMPILER=g++-13 ../
This should execute without any error, now lets build and test the benchmark project:
~/project/gbenchmark $ cmake --build "build-gcc13" --config Release
[ 1%] Building CXX object src/CMakeFiles/benchmark.dir/benchmark.cc.o
[ 2%] Building CXX object src/CMakeFiles/benchmark.dir/benchmark_api_internal.cc.o
...
~/project/gbenchmark $ cmake -E chdir "build-gcc13" ctest --build-config Release
...
100% tests passed, 0 tests failed out of 75
Total Test time (real) = 20.85 sec
clang
And now the same with clang:
~/project/gbenchmark $ cmake -E make_directory "build-clang"
~/project/gbenchmark $ cmake -E chdir "build-clang" cmake -DBENCHMARK_DOWNLOAD_DEPENDENCIES=on
-DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ ../
-- The CXX compiler identification is AppleClang 15.0.0.15000100
...
~/project/gbenchmark $ cmake --build "build-clang" --config Release
[ 1%] Building CXX object src/CMakeFiles/benchmark.dir/benchmark.cc.o
[ 2%] Building CXX object src/CMakeFiles/benchmark.dir/benchmark_api_internal.cc.o
...
~/project/gbenchmark $ cmake -E chdir "build-clang" ctest --build-config Release
...
100% tests passed, 0 tests failed out of 75
Total Test time (real) = 18.93 sec
Of course you could do the same with any other compiler available on your system, but ensure using a seperate build folder for each.
CMake project
Now lets add a CMake project into the project folder, I use cpp-challenge since I will use a task of the CPP challenge in this example.
The content of this folder will be:
/project
/gbenchmark
/cpp-challenge
/CMakeLists.txt
/main.cpp
At first create the cpp-challenge folder and add the CMakeLists.txt file into it:
cmake_minimum_required(VERSION 3.20)
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED True)
# set the project name and main.cpp as the only source file
project(cpp-challenge)
add_executable(cpp-challenge main.cpp)
# now add the benchmark package depending on the used compiler
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
find_package(benchmark REQUIRED PATHS ../gbenchmark/build-gcc13)
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
find_package(benchmark REQUIRED PATHS ../gbenchmark/build-clang)
else()
message(FATAL_ERROR "Unknown compiler '${CMAKE_CXX_COMPILER_ID}'!" )
endif()
# at least lets link with it
target_link_libraries(cpp-challenge benchmark::benchmark)
# and use the include folder from the benchmark package
target_include_directories(benchmark::benchmark INTERFACE ../gbenchmark/include)
Build
Before this step I recommend reading the google benchmark docs.
Add the main.cpp file and put some content in.
I use the cpp challenge task of summarizing each number dividable by 3 and 5 up to a given maximum
and compare the recommended solution from the cpp challenge book (SumOf3and5Until_1) to mine based on
the gauss sum formula (SumOf3and5Until_2).
The benchmark should be performed with a maximum value of 100, 1,000 and 10,000 and the big O complexity should be calculated.
#include <benchmark/benchmark.h>
size_t SumOf3and5Until_1(size_t maxval) {
size_t sum = 0;
for(size_t i = 0; i < maxval; ++i) {
sum += (i % 3 == 0 || i % 5 == 0) ? i : 0;
}
return sum;
}
static void BM_SumOf3and5Until_1(benchmark::State& state) {
for (auto _ : state)
SumOf3and5Until_1(state.range(0));
state.SetComplexityN(state.range(0));
}
BENCHMARK(BM_SumOf3and5Until_1)->Arg(100)->Arg(1000)->Arg(10000)->Complexity();
size_t SumOf3and5Until_2(size_t maxval) {
auto gauss_sum=[](size_t n) { return n * (n+1) / 2;};
--maxval;
return gauss_sum(maxval / 3) * 3 +
gauss_sum(maxval / 5) * 5 -
gauss_sum(maxval / 15) * 15;
}
static void BM_SumOf3and5Until_2(benchmark::State& state) {
for (auto _ : state)
SumOf3and5Until_2(state.range(0));
state.SetComplexityN(state.range(0));
}
BENCHMARK(BM_SumOf3and5Until_2)->Arg(100)->Arg(1000)->Arg(10000)->Complexity();
BENCHMARK_MAIN();
Lets create the project files, first using gcc:
~/project/cpp-challenge $ cmake -E chdir "build-gcc"
cmake DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=gcc-13 -DCMAKE_CXX_COMPILER=g++-13 ../
~/project/cpp-challenge $ cmake --build "build-gcc13" --config Release
Consolidate compiler generated dependencies of target cpp-challenge
[ 50%] Building CXX object CMakeFiles/cpp-challenge.dir/main.cpp.o
[100%] Linking CXX executable cpp-challenge
[100%] Built target cpp-challenge
then clang:
~/project/cpp-challenge $ cmake -E chdir "build-clang"
cmake DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ ../
~/project/cpp-challenge $ cmake --build "build-clang" --config Release
Consolidate compiler generated dependencies of target cpp-challenge
[ 50%] Building CXX object CMakeFiles/cpp-challenge.dir/main.cpp.o
[100%] Linking CXX executable cpp-challenge
[100%] Built target cpp-challenge
Lets benchmark!
Now we’re ready to run the benchmark!
Simply execute the cpp-challenge binary from the folder you created for your compiler:
~/project/cpp-challenge $ build-gcc13/cpp-challenge
Running build-gcc13/cpp-challenge
Run on (4 X 2300 MHz CPU s)
CPU Caches:
L1 Data 32 KiB
L1 Instruction 32 KiB
L2 Unified 256 KiB (x2)
L3 Unified 4096 KiB
Load Average: 4.36, 3.00, 2.62
---------------------------------------------------------------------
Benchmark Time CPU Iterations
---------------------------------------------------------------------
BM_SumOf3and5Until_1/100 434 ns 432 ns 1612807
BM_SumOf3and5Until_1/1000 3960 ns 3957 ns 174557
BM_SumOf3and5Until_1/10000 40827 ns 40727 ns 17185
BM_SumOf3and5Until_1_BigO 4.08 N 4.07 N
BM_SumOf3and5Until_2/100 19.0 ns 19.0 ns 36121389
BM_SumOf3and5Until_2/1000 19.0 ns 19.0 ns 36506629
BM_SumOf3and5Until_2/10000 19.0 ns 19.0 ns 36438975
BM_SumOf3and5Until_2_BigO 19.01 (1) 18.99 (1)
or with clang:
~/project/cpp-challenge $ build-clang/cpp-challenge
Running build-clang/cpp-challenge
Run on (4 X 2300 MHz CPU s)
CPU Caches:
L1 Data 32 KiB
L1 Instruction 32 KiB
L2 Unified 256 KiB (x2)
L3 Unified 4096 KiB
Load Average: 1.94, 2.54, 2.92
---------------------------------------------------------------------
Benchmark Time CPU Iterations
---------------------------------------------------------------------
BM_SumOf3and5Until_1/100 1708 ns 1702 ns 411088
BM_SumOf3and5Until_1/1000 16925 ns 16892 ns 41365
BM_SumOf3and5Until_1/10000 168096 ns 167812 ns 4134
BM_SumOf3and5Until_1_BigO 16.81 N 16.78 N
BM_SumOf3and5Until_2/100 44.3 ns 44.1 ns 15916579
BM_SumOf3and5Until_2/1000 44.1 ns 44.0 ns 15932737
BM_SumOf3and5Until_2/10000 44.0 ns 44.0 ns 15872980
BM_SumOf3and5Until_2_BigO 44.14 (1) 44.03 (1)
Don’t forget to (re)build your binaries in case of changes before running it, best is to put those lines in a bash script and use it for build and execution:
#!/usr/bin/bash
cmake --build "build-gcc" --config Release && cmake --build "build-clang" --config Release
echo 'Apple Clang'
echo '-----------'
build-clang/cpp-challenge
echo 'GNUCC 13.2'
echo '----------'
build-gcc/cpp-challenge