diff --git a/configuration/builders/sequences/sanitizers.py b/configuration/builders/sequences/sanitizers.py index a2d0ac520..6be395bdf 100644 --- a/configuration/builders/sequences/sanitizers.py +++ b/configuration/builders/sequences/sanitizers.py @@ -180,3 +180,130 @@ def asan_ubsan( sequence.add_step(step) return sequence + + +def msan( + config: DockerConfig, + jobs: int, + isDebugBuildType: bool, +): + sequence = BuildSequence() + + sequence.add_step(ShellStep(command=PrintEnvironmentDetails())) + + sequence.add_step( + InContainer( + docker_environment=config, + container_commit=False, + step=ShellStep( + command=FetchTarball(workdir=PurePath("src")), + options=StepOptions(descriptionDone="Fetch tarball"), + ), + ) + ) + + flags = [ + CMakeOption(WITH.MSAN, True), + CMakeOption( + CMAKE.EXE_LINKER_FLAGS, "-L${MSAN_LIBDIR} -Wl,-rpath=${MSAN_LIBDIR}" + ), + CMakeOption( + CMAKE.MODULE_LINKER_FLAGS, "-L${MSAN_LIBDIR} -Wl,-rpath=${MSAN_LIBDIR}" + ), + CMakeOption(WITH.UNIT_TESTS, False), + CMakeOption(WITH.ZLIB, "bundled"), + CMakeOption(WITH.SYSTEMD, "no"), + CMakeOption(PLUGIN.COLUMNSTORE_STORAGE_ENGINE, False), + CMakeOption(PLUGIN.SPIDER_STORAGE_ENGINE, isDebugBuildType), + CMakeOption(PLUGIN.ROCKSDB_STORAGE_ENGINE, False), + CMakeOption(PLUGIN.OQGRAPH_STORAGE_ENGINE, False), + ] + if isDebugBuildType: + flags.append(CMakeOption(CMAKE.BUILD_TYPE, BuildType.DEBUG)) + flags.append(CMakeOption(WITH.DBUG_TRACE, False)) + + sequence.add_step( + InContainer( + docker_environment=config, + step=ShellStep( + command=ConfigureMariaDBCMake( + name="configure", + workdir=PurePath("bld"), + cmake_generator=CMakeGenerator( + use_ccache=True, + flags=flags, + source_path="../src", + compiler=ClangCompiler(), + ), + ), + options=StepOptions(descriptionDone="Configure"), + ), + ) + ) + + sequence.add_step( + InContainer( + docker_environment=config, + step=ShellStep( + command=CompileCMakeCommand( + builddir="bld", + jobs=jobs, + verbose=True, + ), + options=StepOptions(descriptionDone="compile"), + ), + ) + ) + + env_vars = [ + ( + "MSAN_OPTIONS", + "abort_on_error=1:poison_in_dtor=0", + ), + ("MTR_FEEDBACK_PLUGIN", "1"), + ] + + ## ADD MTR TESTS + steps = get_mtr_normal_steps( + jobs=jobs, + env_vars=env_vars, + halt_on_failure=False, + path_to_test_runner=PurePath("bld", "mysql-test"), + additional_mtr_options=[MTROption(MTR.BIG_TEST, True)], + step_wrapping_fn=lambda step: InContainer(docker_environment=config, step=step), + ) + get_mtr_s3_steps( + jobs=jobs, + env_vars=env_vars, + halt_on_failure=False, + additional_mtr_options=[MTROption(MTR.BIG_TEST, True)], + path_to_test_runner=PurePath("bld", "mysql-test"), + step_wrapping_fn=lambda step: InContainer(docker_environment=config, step=step), + ) + if isDebugBuildType: + steps += get_mtr_spider_steps( + jobs=jobs, + env_vars=env_vars, + halt_on_failure=False, + additional_mtr_options=[MTROption(MTR.BIG_TEST, True)], + path_to_test_runner=PurePath("bld", "mysql-test"), + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step + ), + ) + steps += [ + save_mtr_logs( + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step + ), + ), + mtr_junit_reporter( + step_wrapping_fn=lambda step: InContainer( + docker_environment=config, step=step + ), + ), + ] + + for step in steps: + sequence.add_step(step) + + return sequence diff --git a/configuration/steps/generators/cmake/options.py b/configuration/steps/generators/cmake/options.py index 94aca5085..b7e5bfdef 100644 --- a/configuration/steps/generators/cmake/options.py +++ b/configuration/steps/generators/cmake/options.py @@ -24,8 +24,10 @@ class CMAKE(StrEnum): C_FLAGS = "C_FLAGS" C_COMPILER_LAUNCHER = "C_COMPILER_LAUNCHER" CXX_COMPILER_LAUNCHER = "CXX_COMPILER_LAUNCHER" + EXE_LINKER_FLAGS = "EXE_LINKER_FLAGS" INSTALL_PREFIX = "INSTALL_PREFIX" LIBRARY_PATH = "LIBRARY_PATH" + MODULE_LINKER_FLAGS = "MODULE_LINKER_FLAGS" def __str__(self): return f"CMAKE_{self.value}" @@ -71,13 +73,16 @@ class WITH(StrEnum): EMBEDDED_SERVER = "EMBEDDED_SERVER" EXTRA_CHARSETS = "EXTRA_CHARSETS" JEMALLOC = "JEMALLOC" + MSAN = "MSAN" NONE = "NONE" SAFEMALLOC = "SAFEMALLOC" SSL = "SSL" + SYSTEMD = "SYSTEMD" UBSAN = "UBSAN" UNIT_TESTS = "UNIT_TESTS" VALGRIND = "VALGRIND" WSREP = "WSREP" + ZLIB = "ZLIB" def __str__(self): return f"WITH_{self.value}" diff --git a/constants.py b/constants.py index 22e121d24..5d059f3e3 100644 --- a/constants.py +++ b/constants.py @@ -177,6 +177,7 @@ "amd64-debian-12-debug-embedded", "amd64-fedora-41", "amd64-fedora-42", + "amd64-msan-clang-20-debug", "amd64-opensuse-1506", "amd64-rhel-10", "amd64-sles-1506", diff --git a/master-docker-nonstandard-2/master.cfg b/master-docker-nonstandard-2/master.cfg index ce1a52ee1..6c22dd3e7 100644 --- a/master-docker-nonstandard-2/master.cfg +++ b/master-docker-nonstandard-2/master.cfg @@ -103,15 +103,15 @@ for full_test_worker in ["debian-12-32-bit", "debian-12-32-bit-debug"]: shm_size="30G", ) -for msan_worker in ["msan-clang-20", "msan-clang-20-debug"]: - for worker_id in [1,4,5]: # Hetzner designated workers for protected builds - addWorker( - "hz-bbw", - worker_id, - msan_worker, - os.environ["CONTAINER_REGISTRY_URL"] + "debian12-msan-clang-20", - jobs=20, - save_packages=False, +msan_worker = "msan-clang-20" +for worker_id in [1,4,5]: # Hetzner designated workers for protected builds + addWorker( + "hz-bbw", + worker_id, + msan_worker, + os.environ["CONTAINER_REGISTRY_URL"] + "debian12-msan-clang-20", + jobs=20, + save_packages=False, ) addWorker( @@ -696,20 +696,6 @@ c["builders"].append( ) ) -c["builders"].append( - util.BuilderConfig( - name="amd64-msan-clang-20-debug", - workernames=workers["x64-bbw-docker-msan-clang-20-debug"], - tags=["Debian", "clang", "msan", "debug"], - collapseRequests=True, - nextBuild=nextBuild, - canStartBuild=canStartBuild, - properties={"c_compiler": "clang", "cxx_compiler": "clang++", "build_type": "Debug"}, - locks=getLocks, - factory=f_msan_build, - ) -) - c["builders"].append( util.BuilderConfig( name="amd64-ubuntu-2204-jepsen-mariadb", diff --git a/master-migration/master.cfg b/master-migration/master.cfg index 11f884ec2..ca3760b76 100644 --- a/master-migration/master.cfg +++ b/master-migration/master.cfg @@ -15,7 +15,7 @@ from configuration.builders.sequences.compile_only import ( ) from configuration.builders.sequences.debug import openssl_fips from configuration.builders.sequences.release import deb_autobake, rpm_autobake -from configuration.builders.sequences.sanitizers import asan_ubsan +from configuration.builders.sequences.sanitizers import asan_ubsan, msan from configuration.reporters import github_summary from configuration.workers import worker from master_common import base_master_config, IS_CHECKCONFIG @@ -33,10 +33,10 @@ c = BuildmasterConfig = base_master_config(config) WORKER_POOL = worker.WorkerPool() WORKER_POOL.add( - arch="amd64", worker=worker.NonLatent(name="hz-bbw8", config=config, total_jobs=96) + arch="amd64", worker=worker.NonLatent(name="hz-bbw8", config=config, total_jobs=110) ) WORKER_POOL.add( - arch="amd64", worker=worker.NonLatent(name="hz-bbw9", config=config, total_jobs=96) + arch="amd64", worker=worker.NonLatent(name="hz-bbw9", config=config, total_jobs=110) ) @@ -276,6 +276,47 @@ for builder in ["amd64-ubasan-clang-20", "amd64-ubasan-clang-20-debug"]: c["builders"].append(ubasan_builder(name=builder, debug=builder.endswith("debug"))) +def msan_builder(name: str, debug: bool) -> GenericBuilder: + tags_msan = ("Debian", "clang", "msan", "big") + jobs = 12 + if debug: + tag_msan = tags_msan + ("debug",) + jobs = 20 + + return GenericBuilder( + name=name, + sequences=[ + msan( + jobs=jobs, + config=DockerConfig( + repository=os.environ["CONTAINER_REGISTRY_URL"], + image_tag="debian12-msan-clang-20", + workdir=PurePath("/home/buildbot"), + bind_mounts=[ + ("/srv/buildbot/ccache", "/mnt/ccache"), + (f'{os.environ["MASTER_PACKAGES_DIR"]}/', "/packages"), + ], + shm_size="24g", + env_vars=[ + ("CCACHE_DIR", "/mnt/ccache"), + ("ARTIFACTS_URL", os.environ["ARTIFACTS_URL"]), + ], + memlock_limit=memlock_limit, + ), + isDebugBuildType=debug, + ) + ], + ).get_config( + workers=WORKER_POOL.get_workers_for_arch(arch="amd64"), + next_build=nextBuild, + can_start_build=canStartBuild, + tags=list(tags_msan), + jobs=jobs, + ) + + +builder = "amd64-msan-clang-20-debug" +c["builders"].append(msan_builder(name=builder, debug=builder.endswith("debug"))) ## ------------------------------------------------------------------- ## ## REPORTERS ##