Merge pull request #17 from itzmeanjan/modernize

Modernize Testing Infrastructure
itzmeanjan · Feb 10, 2024 · 4208b18 · 4208b18
2 parents f8ce23d + 597bb37
commit 4208b18
Show file tree

Hide file tree

Showing 28 changed files with 1,018 additions and 825 deletions.
diff --git a/.clang-format b/.clang-format
@@ -81,7 +81,7 @@ BreakBeforeTernaryOperators: true
 BreakConstructorInitializers: BeforeComma
 BreakInheritanceList: BeforeComma
 BreakStringLiterals: true
-ColumnLimit:     80
+ColumnLimit:     172
 CommentPragmas:  '^ IWYU pragma:'
 CompactNamespaces: false
 ConstructorInitializerIndentWidth: 2

diff --git a/.github/workflows/test_ci.yml b/.github/workflows/test_ci.yml
@@ -1,4 +1,4 @@
-name: Test FrodoKEM using CI
+name: Test FrodoKEM
 
 on:
   push:
@@ -8,19 +8,19 @@ on:
 
 jobs:
   build:
-
-    runs-on: ubuntu-22.04
+    runs-on: ${{matrix.os}}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, # x86_64
+             macos-14       # Apple Silicon
+            ]
 
     steps:
-    - uses: actions/checkout@v3
-    - name : Fetch Dependency
-      run: git submodule update --init
-    - name: Get CMake
-      run: sudo apt-get install cmake
+    - uses: actions/checkout@v4
     - name: Setup Google-Test
       run: |
         pushd ~
-        git clone https://github.com/google/googletest.git -b v1.13.0
+        git clone https://github.com/google/googletest.git -b v1.14.0
         pushd googletest
         mkdir build
         pushd build
@@ -30,13 +30,9 @@ jobs:
         popd
         popd
         popd
-    - name: Execute Tests
+    - name: Execute Tests on ${{matrix.os}}
       run: make -j
-    - name: Run Examples
-      run: |
-        g++ -std=c++20 -O3 -march=native -Wall -I include -I sha3/include -I subtle/include examples/efrodo640_kem.cpp
-        ./a.out
-        g++ -std=c++20 -O3 -march=native -Wall -I include -I sha3/include -I subtle/include examples/frodo640_kem.cpp
-        ./a.out
-    - name: Cleanup
-      run: make clean
+    - name: Execute Tests with AddressSanitizer on ${{matrix.os}}
+      run: make asan_test -j
+    - name: Execute Tests with UndefinedBehaviourSanitizer on ${{matrix.os}}
+      run: make ubsan_test -j
diff --git a/.gitmodules b/.gitmodules
@@ -4,3 +4,9 @@
 [submodule "subtle"]
 	path = subtle
 	url = https://github.com/itzmeanjan/subtle.git
+[submodule "gtest-parallel"]
+	path = gtest-parallel
+	url = https://github.com/google/gtest-parallel.git
+[submodule "dudect"]
+	path = dudect
+	url = https://github.com/oreparaz/dudect.git
diff --git a/Makefile b/Makefile
@@ -1,23 +1,42 @@
-CXX = g++
+CXX ?= clang++
 CXX_FLAGS = -std=c++20
 WARN_FLAGS = -Wall -Wextra -pedantic
 OPT_FLAGS = -O3 -march=native
 LINK_FLAGS = -flto
+ASAN_FLAGS = -g -O1 -fno-omit-frame-pointer -fno-optimize-sibling-calls -fsanitize=address # From https://clang.llvm.org/docs/AddressSanitizer.html
+UBSAN_FLAGS = -g -O1 -fno-omit-frame-pointer -fno-optimize-sibling-calls -fsanitize=undefined # From https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html
+
+SHA3_INC_DIR = ./sha3/include
+SUBTLE_INC_DIR = ./subtle/include
+DUDECT_INC_DIR = ./dudect/src
 I_FLAGS = -I ./include
-DEP_IFLAGS = -I ./sha3/include -I ./subtle/include
+DEP_IFLAGS = -I $(SHA3_INC_DIR) -I $(SUBTLE_INC_DIR)
+DUDECT_DEP_IFLAGS = $(DEP_IFLAGS) -I $(DUDECT_INC_DIR)
 
 SRC_DIR = include
 FRODO_SOURCES := $(wildcard $(SRC_DIR)/*.hpp)
 BUILD_DIR = build
+ASAN_BUILD_DIR = $(BUILD_DIR)/asan
+UBSAN_BUILD_DIR = $(BUILD_DIR)/ubsan
+DUDECT_BUILD_DIR = $(BUILD_DIR)/dudect
 
 TEST_DIR = tests
+DUDECT_TEST_DIR = $(TEST_DIR)/dudect
 TEST_SOURCES := $(wildcard $(TEST_DIR)/*.cpp)
+DUDECT_TEST_SOURCES := $(wildcard $(DUDECT_TEST_DIR)/*.cpp)
 TEST_OBJECTS := $(addprefix $(BUILD_DIR)/, $(notdir $(patsubst %.cpp,%.o,$(TEST_SOURCES))))
+ASAN_TEST_OBJECTS := $(addprefix $(ASAN_BUILD_DIR)/, $(notdir $(patsubst %.cpp,%.o,$(TEST_SOURCES))))
+UBSAN_TEST_OBJECTS := $(addprefix $(UBSAN_BUILD_DIR)/, $(notdir $(patsubst %.cpp,%.o,$(TEST_SOURCES))))
 TEST_LINK_FLAGS = -lgtest -lgtest_main
 TEST_BINARY = $(BUILD_DIR)/test.out
+ASAN_TEST_BINARY = $(ASAN_BUILD_DIR)/test.out
+UBSAN_TEST_BINARY = $(UBSAN_BUILD_DIR)/test.out
+DUDECT_TEST_BINARIES := $(addprefix $(DUDECT_BUILD_DIR)/, $(notdir $(patsubst %.cpp,%.out,$(DUDECT_TEST_SOURCES))))
+GTEST_PARALLEL = ./gtest-parallel/gtest-parallel
 
 BENCHMARK_DIR = benchmarks
 BENCHMARK_SOURCES := $(wildcard $(BENCHMARK_DIR)/*.cpp)
+BENCHMARK_HEADERS := $(wildcard $(BENCHMARK_DIR)/*.hpp)
 BENCHMARK_OBJECTS := $(addprefix $(BUILD_DIR)/, $(notdir $(patsubst %.cpp,%.o,$(BENCHMARK_SOURCES))))
 BENCHMARK_LINK_FLAGS = -lbenchmark -lbenchmark_main
 BENCHMARK_BINARY = $(BUILD_DIR)/bench.out
@@ -29,14 +48,58 @@ all: test
 $(BUILD_DIR):
 	mkdir -p $@
 
-$(BUILD_DIR)/%.o: $(TEST_DIR)/%.cpp $(BUILD_DIR)
+$(ASAN_BUILD_DIR): $(BUILD_DIR)
+	mkdir -p $@
+
+$(UBSAN_BUILD_DIR): $(BUILD_DIR)
+	mkdir -p $@
+
+$(DUDECT_BUILD_DIR): $(BUILD_DIR)
+	mkdir -p $@
+
+$(SHA3_INC_DIR):
+	git submodule update --init
+
+$(SUBTLE_INC_DIR): $(SHA3_INC_DIR)
+	git submodule update --init
+
+$(GTEST_PARALLEL): $(SUBTLE_INC_DIR)
+	git submodule update --init
+
+$(DUDECT_INC_DIR): $(GTEST_PARALLEL)
+	git submodule update --init
+
+$(BUILD_DIR)/%.o: $(TEST_DIR)/%.cpp $(BUILD_DIR) $(SHA3_INC_DIR) $(SUBTLE_INC_DIR)
 	$(CXX) $(CXX_FLAGS) $(WARN_FLAGS) $(OPT_FLAGS) $(I_FLAGS) $(DEP_IFLAGS) -c $< -o $@
 
+$(ASAN_BUILD_DIR)/%.o: $(TEST_DIR)/%.cpp $(ASAN_BUILD_DIR) $(SHA3_INC_DIR) $(SUBTLE_INC_DIR)
+	$(CXX) $(CXX_FLAGS) $(WARN_FLAGS) $(ASAN_FLAGS) $(I_FLAGS) $(DEP_IFLAGS) -c $< -o $@
+
+$(UBSAN_BUILD_DIR)/%.o: $(TEST_DIR)/%.cpp $(UBSAN_BUILD_DIR) $(SHA3_INC_DIR) $(SUBTLE_INC_DIR)
+	$(CXX) $(CXX_FLAGS) $(WARN_FLAGS) $(UBSAN_FLAGS) $(I_FLAGS) $(DEP_IFLAGS) -c $< -o $@
+
 $(TEST_BINARY): $(TEST_OBJECTS)
 	$(CXX) $(OPT_FLAGS) $(LINK_FLAGS) $^ $(TEST_LINK_FLAGS) -o $@
 
-test: $(TEST_BINARY)
-	./$<
+$(ASAN_TEST_BINARY): $(ASAN_TEST_OBJECTS)
+	$(CXX) $(ASAN_FLAGS) $^ $(TEST_LINK_FLAGS) -o $@
+
+$(UBSAN_TEST_BINARY): $(UBSAN_TEST_OBJECTS)
+	$(CXX) $(UBSAN_FLAGS) $^ $(TEST_LINK_FLAGS) -o $@
+
+$(DUDECT_BUILD_DIR)/%.out: $(DUDECT_TEST_DIR)/%.cpp $(DUDECT_BUILD_DIR) $(SHA3_INC_DIR) $(SUBTLE_INC_DIR) $(DUDECT_INC_DIR)
+	$(CXX) $(CXX_FLAGS) $(WARN_FLAGS) $(OPT_FLAGS) $(I_FLAGS) $(DUDECT_DEP_IFLAGS) -lm $(LINK_FLAGS) $< -o $@
+
+test: $(TEST_BINARY) $(GTEST_PARALLEL)
+	$(GTEST_PARALLEL) $< --print_test_times
+
+asan_test: $(ASAN_TEST_BINARY) $(GTEST_PARALLEL)
+	$(GTEST_PARALLEL) $< --print_test_times
+
+ubsan_test: $(UBSAN_TEST_BINARY) $(GTEST_PARALLEL)
+	$(GTEST_PARALLEL) $< --print_test_times
+
+dudect_test_build: $(DUDECT_TEST_BINARIES)
 
 $(BUILD_DIR)/%.o: $(BENCHMARK_DIR)/%.cpp $(BUILD_DIR)
 	$(CXX) $(CXX_FLAGS) $(WARN_FLAGS) $(OPT_FLAGS) $(I_FLAGS) $(DEP_IFLAGS) -c $< -o $@
@@ -46,19 +109,19 @@ $(BENCHMARK_BINARY): $(BENCHMARK_OBJECTS)
 
 benchmark: $(BENCHMARK_BINARY)
 	# Must *not* build google-benchmark with libPFM
-	./$< --benchmark_time_unit=ms --benchmark_min_warmup_time=.1 --benchmark_enable_random_interleaving=true --benchmark_repetitions=8 --benchmark_min_time=0.1s --benchmark_display_aggregates_only=true --benchmark_counters_tabular=true
+	./$< --benchmark_time_unit=ms --benchmark_min_warmup_time=.1 --benchmark_enable_random_interleaving=true --benchmark_repetitions=10 --benchmark_min_time=0.1s --benchmark_display_aggregates_only=true --benchmark_counters_tabular=true
 
 $(PERF_BINARY): $(BENCHMARK_OBJECTS)
 	$(CXX) $(OPT_FLAGS) $(LINK_FLAGS) $^ $(PERF_LINK_FLAGS) -o $@
 
 perf: $(PERF_BINARY)
 	# Must build google-benchmark with libPFM, follow https://gist.github.com/itzmeanjan/05dc3e946f635d00c5e0b21aae6203a7
-	./$< --benchmark_time_unit=ms --benchmark_min_warmup_time=.1 --benchmark_enable_random_interleaving=true --benchmark_repetitions=8 --benchmark_min_time=0.1s --benchmark_display_aggregates_only=true --benchmark_counters_tabular=true --benchmark_perf_counters=CYCLES
+	./$< --benchmark_time_unit=ms --benchmark_min_warmup_time=.1 --benchmark_enable_random_interleaving=true --benchmark_repetitions=10 --benchmark_min_time=0.1s --benchmark_display_aggregates_only=true --benchmark_counters_tabular=true --benchmark_perf_counters=CYCLES
 
 .PHONY: format clean
 
 clean:
 	rm -rf $(BUILD_DIR)
 
-format: $(FRODO_SOURCES) $(TEST_SOURCES) $(BENCHMARK_SOURCES)
+format: $(FRODO_SOURCES) $(TEST_SOURCES) $(BENCHMARK_SOURCES) $(BENCHMARK_HEADERS) $(DUDECT_TEST_SOURCES)
 	clang-format -i $^