From bb1331c0cb148ea9a0b83d115bb6fb9f4a919f68 Mon Sep 17 00:00:00 2001
From: Dale Glass <dale@daleglass.net>
Date: Sun, 5 Nov 2023 19:05:09 +0100
Subject: [PATCH] Fix memory debugging.

* Disables WebRTC (build fails)
* Recommends disabling optimization
* Recommends enabling debugging
* Remove forced optimization from plugins
---
 cmake/macros/MemoryDebugger.cmake     | 15 +++++++++++++++
 libraries/audio-client/CMakeLists.txt |  6 +++++-
 libraries/networking/CMakeLists.txt   |  6 +++++-
 libraries/shared/src/shared/WebRTC.h  |  8 +++++---
 plugins/CMakeLists.txt                |  2 +-
 5 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/cmake/macros/MemoryDebugger.cmake b/cmake/macros/MemoryDebugger.cmake
index abcede8bee..4bbae3d39e 100644
--- a/cmake/macros/MemoryDebugger.cmake
+++ b/cmake/macros/MemoryDebugger.cmake
@@ -15,10 +15,25 @@ if ("$ENV{OVERTE_MEMORY_DEBUGGING}")
     message(FATAL_ERROR "Thread debugging and memory debugging can't be enabled at the same time." )
   endif()
 
+  if (OVERTE_OPTIMIZE)
+    message(WARNING "You should consider building without optimization by passing -DOVERTE_OPTIMIZE=false to CMake")
+  endif()
+  if (NOT CMAKE_BUILD_TYPE MATCHES "Debug")
+    message(WARNING "You should consider building with debugging enabled by passing -DCMAKE_BUILD_TYPE=Debug to CMake. Current type is ${CMAKE_BUILD_TYPE}")
+  endif()
+
+
   SET( OVERTE_MEMORY_DEBUGGING true )
+  SET ( DISABLE_WEBRTC true )
 endif ()
 
 if ( OVERTE_MEMORY_DEBUGGING)
+  # WebRTC doesn't work with memory debugging enabled, it fails to link:
+  # /usr/bin/ld: ../../libraries/networking/libnetworking.so: undefined reference to `typeinfo for rtc::Thread'
+  # /usr/bin/ld: ../../libraries/networking/libnetworking.so: undefined reference to `typeinfo for webrtc::SessionDescriptionInterface'
+  # /usr/bin/ld: ../../libraries/networking/libnetworking.so: undefined reference to `typeinfo for webrtc::IceCandidateInterface'
+  add_compile_definitions(DISABLE_WEBRTC)
+
   if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
     SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -fsanitize=undefined -fsanitize=address -fsanitize-recover=address")
     SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -fsanitize=address -fsanitize-recover=address")
diff --git a/libraries/audio-client/CMakeLists.txt b/libraries/audio-client/CMakeLists.txt
index 2f35b227fd..07ed155033 100644
--- a/libraries/audio-client/CMakeLists.txt
+++ b/libraries/audio-client/CMakeLists.txt
@@ -14,7 +14,11 @@ include_hifi_library_headers(script-engine)
 
 if (ANDROID)
 else ()
-  target_webrtc()
+  if (NOT DISABLE_WEBRTC )
+    target_webrtc()
+  else()
+    message(WARNING "WebRTC is supported on this platform but has been disabled for this build (likely memory debugging)")
+  endif()
 endif ()
 
 # append audio includes to our list of includes to bubble
diff --git a/libraries/networking/CMakeLists.txt b/libraries/networking/CMakeLists.txt
index 1d46236e6c..3550cf424e 100644
--- a/libraries/networking/CMakeLists.txt
+++ b/libraries/networking/CMakeLists.txt
@@ -8,7 +8,11 @@ add_crashpad()
 target_breakpad()
 
 if (WIN32 OR (UNIX AND NOT APPLE))
-    target_webrtc()
+    if (NOT DISABLE_WEBRTC )
+        target_webrtc()
+    else()
+        message(WARNING "WebRTC is supported on this platform but has been disabled for this build (likely memory debugging)")
+    endif()
 endif ()
 
 if (WIN32)
diff --git a/libraries/shared/src/shared/WebRTC.h b/libraries/shared/src/shared/WebRTC.h
index ef194e971a..9f3c954b60 100644
--- a/libraries/shared/src/shared/WebRTC.h
+++ b/libraries/shared/src/shared/WebRTC.h
@@ -37,9 +37,11 @@
 // #  define WEBRTC_POSIX 1
 // #  define WEBRTC_LEGACY 1
 #elif defined(Q_OS_LINUX) && defined(Q_PROCESSOR_X86_64)
-#  define WEBRTC_AUDIO 1
-#  define WEBRTC_POSIX 1
-#  define WEBRTC_DATA_CHANNELS 1
+#  ifndef DISABLE_WEBRTC
+#    define WEBRTC_AUDIO 1
+#    define WEBRTC_POSIX 1
+#    define WEBRTC_DATA_CHANNELS 1
+#  endif
 #elif defined(Q_OS_LINUX) && defined(Q_PROCESSOR_ARM)
 // WebRTC is basically impossible to build on aarch64 Linux.
 // I am looking at https://gitlab.freedesktop.org/pulseaudio/webrtc-audio-processing for an alternative.
diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt
index 35a9457f3e..6f2bd56212 100644
--- a/plugins/CMakeLists.txt
+++ b/plugins/CMakeLists.txt
@@ -9,7 +9,7 @@
 # add the plugin directories
 file(GLOB PLUGIN_SUBDIRS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/*")
 list(REMOVE_ITEM PLUGIN_SUBDIRS "CMakeFiles")
-set(CMAKE_BUILD_TYPE "Release")
+
 # client-side plugins
 if (NOT SERVER_ONLY AND NOT ANDROID)
   if (NOT CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")