Fix DTMF minimums, standalone ICE gathering, end-of-candidates reuse and audio capture time

python-webrtc/cpp/NATIVE.md

@@ -102,7 +102,7 @@ One direction: Python → C++ → libwebrtc.
   unregistered is ended by `Disarm()` at the start of the destructor, which then waits for the callback's thread once.
 - Raw pointers never own: they point to libwebrtc observers and callbacks, or into C APIs. A raw pointer libwebrtc
   hands over to own (`OnSuccess(SessionDescriptionInterface *)`) goes into a `std::unique_ptr` right away.
-- An event's arguments are native values copied at `Emit`; they're converted to Python on the loop thread.
+- An event's arguments are native values copied or moved at `Emit`; they're converted to Python on the loop thread.
 - Registries and the Dispatcher are leaked on purpose (`static auto *x = new ...`, `ForkLocal`): native threads can
   outlive static destructors at exit.
 

python-webrtc/cpp/src/interfaces/peer_connection_factory.cpp

@@ -17,7 +17,6 @@
 #include "../utils/port_allocator.h"
 #include "../utils/task_queue_factory.h"
 
-#include <api/audio/builtin_audio_processing_builder.h>
 #include <api/audio_codecs/builtin_audio_decoder_factory.h>
 #include <api/audio_codecs/builtin_audio_encoder_factory.h>
 #include <api/create_modular_peer_connection_factory.h>
@@ -84,7 +83,7 @@ namespace python_webrtc {
     }
   } // namespace
 
-  // webrtc::CreatePeerConnectionFactory without the RTC event log, which keeps every packet's buffer in memory
+  // webrtc::CreatePeerConnectionFactory without the RTC event log (holds every packet) or audio processing (no capture)
   webrtc::PeerConnectionFactoryDependencies PeerConnectionFactory::Dependencies() {
     webrtc::PeerConnectionFactoryDependencies dependencies;
     dependencies.network_thread = _workerThread.get();
@@ -95,7 +94,6 @@ namespace python_webrtc {
     dependencies.adm = _audioDeviceModule;
     dependencies.audio_encoder_factory = webrtc::CreateBuiltinAudioEncoderFactory();
     dependencies.audio_decoder_factory = webrtc::CreateBuiltinAudioDecoderFactory();
-    dependencies.audio_processing_builder = std::make_unique<webrtc::BuiltinAudioProcessingBuilder>();
     dependencies.video_encoder_factory = std::make_unique<VideoEncoderFactory>();
     dependencies.video_decoder_factory = std::make_unique<VideoDecoderFactory>();
     webrtc::EnableMedia(dependencies);
@@ -117,9 +115,8 @@ namespace python_webrtc {
 
   PeerConnectionFactory::PeerConnectionFactory()
       : _fieldTrials(_trials), _networkIgnoreMask(_loopbackAllowed ? 0 : webrtc::kDefaultNetworkIgnoreMask) {
-    _workerThread = Started(std::make_unique<webrtc::Thread>(std::make_unique<NetworkSocketServer>()),
-                            "PeerConnectionFactory:workerThread");
-    _signalingThread = Started(webrtc::Thread::Create(), "PeerConnectionFactory:signalingThread");
+    _workerThread = Started(std::make_unique<webrtc::Thread>(std::make_unique<NetworkSocketServer>()), "wrtc-network");
+    _signalingThread = Started(webrtc::Thread::Create(), "wrtc-signaling");
     // sanitizer builds check that these threads never enter Python
     _workerThread->PostTask(&TagNativeThread);
     _signalingThread->PostTask(&TagNativeThread);

python-webrtc/cpp/src/interfaces/peer_connection_factory.h

@@ -66,6 +66,11 @@ namespace python_webrtc {
     std::unique_ptr<webrtc::PortAllocator>
     CreatePortAllocator(const webrtc::PeerConnectionInterface::RTCConfiguration &configuration);
 
+    [[nodiscard]] std::shared_ptr<webrtc::BasicNetworkManager> networkManager() const { return _networkManager; }
+    [[nodiscard]] std::shared_ptr<webrtc::BasicPacketSocketFactory> packetSocketFactory() const {
+      return _packetSocketFactory;
+    }
+
   private:
     webrtc::PeerConnectionFactoryDependencies Dependencies();
 

python-webrtc/cpp/src/interfaces/rtc_audio_track_source.cpp

@@ -12,6 +12,8 @@
 #include <cstdint>
 #include <vector>
 
+#include <rtc_base/time_utils.h>
+
 namespace python_webrtc {
 
   webrtc::MediaSourceInterface::SourceState RTCAudioTrackSource::state() const {
@@ -42,10 +44,10 @@ namespace python_webrtc {
   }
 
   void RTCAudioTrackSource::PushSamples(const void *samples, int bitsPerSample, int sampleRate, size_t channels,
-                                        size_t frames) {
+                                        size_t frames, int64_t captureTimeMs) {
     const std::scoped_lock lock(_sinkMutex);
     for (auto *sink : _sinks) {
-      sink->OnData(samples, bitsPerSample, sampleRate, channels, frames, std::nullopt);
+      sink->OnData(samples, bitsPerSample, sampleRate, channels, frames, captureTimeMs);
     }
   }
 
@@ -65,7 +67,7 @@ namespace python_webrtc {
         seed = (seed * multiplier) + increment;
         sample = static_cast<int16_t>((static_cast<int32_t>(seed >> seedShift) % (2 * amplitude)) - amplitude);
       }
-      PushSamples(samples.data(), bitsPerSample, sampleRate, 1, frames);
+      PushSamples(samples.data(), bitsPerSample, sampleRate, 1, frames, webrtc::TimeMillis());
     };
     _control = std::make_shared<SourceControl>();
     _control->microphone = true;

python-webrtc/cpp/src/interfaces/rtc_audio_track_source.h

@@ -9,6 +9,7 @@
 #define PYTHON_WEBRTC_INTERFACES_RTC_AUDIO_TRACK_SOURCE_H_
 
 #include <atomic>
+#include <cstdint>
 #include <memory>
 #include <mutex>
 #include <vector>
@@ -35,7 +36,8 @@ namespace python_webrtc {
 
     bool remote() const override;
 
-    void PushSamples(const void *samples, int bitsPerSample, int sampleRate, size_t channels, size_t frames);
+    void PushSamples(const void *samples, int bitsPerSample, int sampleRate, size_t channels, size_t frames,
+                     int64_t captureTimeMs);
 
     // ends the tracks of the source; must be called on the signaling thread, where they observe it
     void End();

python-webrtc/cpp/src/interfaces/rtc_data_channel.cpp

@@ -14,6 +14,7 @@
 #include "../exceptions.h"
 #include "../utils/gil.h"
 #include "../utils/libwebrtc_thread.h"
+#include "../utils/parking.h"
 
 namespace python_webrtc {
 
@@ -61,12 +62,13 @@ namespace python_webrtc {
   void RTCDataChannel::Init(pybind11::module &m) {
     pybind11::class_<DataChannelMessage>(m, "DataChannelMessage")
         .def_property_readonly("data", [](const DataChannelMessage &message) -> pybind11::object {
+          const auto *data = message.data.cdata<char>();
           if (message.binary) {
-            return pybind11::bytes(message.data);
+            return pybind11::bytes(data, message.data.size());
           }
           // the upper layers verify UTF-8, as libwebrtc doesn't
           return pybind11::reinterpret_steal<pybind11::object>(
-              PyUnicode_DecodeUTF8(message.data.data(), static_cast<Py_ssize_t>(message.data.size()), "replace"));
+              PyUnicode_DecodeUTF8(data, static_cast<Py_ssize_t>(message.data.size()), "replace"));
         });
 
     DefineBinding(pybind11::class_<RTCDataChannel, Binding, std::shared_ptr<RTCDataChannel>>(m, "RTCDataChannel"))
@@ -84,7 +86,15 @@ namespace python_webrtc {
         .def_property("bufferedAmountLowThreshold", nogil_fn(&RTCDataChannel::GetBufferedAmountLowThreshold),
                       nogil_fn(&RTCDataChannel::SetBufferedAmountLowThreshold))
         .def_property("binaryType", nogil_fn(&RTCDataChannel::GetBinaryType), nogil_fn(&RTCDataChannel::SetBinaryType))
-        .def("send", &RTCDataChannel::Send, nogil(), pybind11::arg("data"), pybind11::arg("binary"))
+        .def(
+            "send",
+            // only bytes: the view must outlive the call without the GIL, which a bytearray's resize breaks
+            [](RTCDataChannel &self, const pybind11::bytes &data, bool binary) {
+              const std::string_view view = data;
+              const gil_release release;
+              return self.Send(view, binary);
+            },
+            pybind11::arg("data"), pybind11::arg("binary"))
         .def("close", &RTCDataChannel::Close, nogil())
         .def("_surfaceState", &RTCDataChannel::SurfaceState, nogil(), pybind11::arg("state"))
         .def("_decreaseBufferedAmount", &RTCDataChannel::DecreaseBufferedAmount, nogil(), pybind11::arg("sent"));
@@ -116,17 +126,22 @@ namespace python_webrtc {
       Emit("open", state);
       break;
     case DataState::kClosing:
+      Parking::At("datachannel.closing");
       Emit("closing", state);
       break;
     case DataState::kClosed: {
       auto error = _channel->error();
-      // closing locally isn't an error, even if queued messages couldn't be sent
       bool closedLocally = false;
       {
         const std::scoped_lock lock(_closeMutex);
         closedLocally = _closeRequested;
       }
-      if (!error.ok() && !closedLocally) {
+      const bool detailed = error.error_detail() != webrtc::RTCErrorDetailType::NONE;
+      // closing locally isn't an error
+      if (!error.ok() && (detailed || !closedLocally)) {
+        if (!detailed) {
+          error.set_error_detail(webrtc::RTCErrorDetailType::DATA_CHANNEL_FAILURE);
+        }
         Emit("error", RTCCallbackException(std::move(error)));
       }
       Emit("close", state);
@@ -141,8 +156,7 @@ namespace python_webrtc {
   }
 
   void RTCDataChannel::OnMessage(const webrtc::DataBuffer &buffer) {
-    Emit("message",
-         DataChannelMessage{.data = std::string(buffer.data.cdata<char>(), buffer.size()), .binary = buffer.binary});
+    Emit("message", DataChannelMessage{.data = buffer.data, .binary = buffer.binary});
   }
 
   void RTCDataChannel::OnBufferedAmountChange(uint64_t sentDataSize) {
@@ -204,7 +218,7 @@ namespace python_webrtc {
   }
 
   webrtc::Priority RTCDataChannel::GetPriority() {
-    // the ranges Chromium maps priority values to
+    // splits at the midpoints between the DCEP priority values of W3C webrtc-priority (128, 256, 512, 1024)
     constexpr int veryLowMax = 192;
     constexpr int lowMax = 384;
     constexpr int mediumMax = 768;
@@ -277,7 +291,7 @@ namespace python_webrtc {
 
   } // namespace
 
-  bool RTCDataChannel::Send(const std::string &data, bool binary) {
+  bool RTCDataChannel::Send(std::string_view data, bool binary) {
     if (GetReadyState() != DataState::kOpen) {
       throw RTCException(webrtc::RTCErrorType::INVALID_STATE, "RTCDataChannel.readyState is not 'open'");
     }
@@ -307,14 +321,8 @@ namespace python_webrtc {
   }
 
   void RTCDataChannel::SendNow(webrtc::DataBuffer buffer) {
-    auto size = buffer.size();
-    // posted to the network thread, in call order; a failed send closes the channel
-    _channel->SendAsync(std::move(buffer), [weak = weak_from_this(), size](const webrtc::RTCError &error) {
-      auto self = weak.lock();
-      if (!error.ok() && self) {
-        self->DecreaseBufferedAmount(size);
-      }
-    });
+    // posted in call order; libwebrtc reports a failed send's bytes through OnBufferedAmountChange too
+    _channel->SendAsync(std::move(buffer), nullptr);
   }
 
   void RTCDataChannel::SetStateLocked(DataState state) {

python-webrtc/cpp/src/interfaces/rtc_data_channel.h

@@ -15,8 +15,10 @@
 #include <mutex>
 #include <optional>
 #include <string>
+#include <string_view>
 
 #include <api/data_channel_interface.h>
+#include <rtc_base/copy_on_write_buffer.h>
 
 #include <pybind11/pybind11.h>
 
@@ -32,7 +34,8 @@ namespace python_webrtc {
 
   // A received message, converted to str or bytes by Python
   struct DataChannelMessage {
-    std::string data;
+    // refcounted and only read, so it's shared across threads without a copy
+    webrtc::CopyOnWriteBuffer data;
     bool binary;
   };
 
@@ -113,8 +116,8 @@ namespace python_webrtc {
     // whether bufferedAmount dropped to the threshold
     bool DecreaseBufferedAmount(uint64_t sent);
 
-    // whether the message is the only one queued: the channel's activity changes only then
-    bool Send(const std::string &data, bool binary);
+    // whether the message is the only one queued: the channel's activity changes only then; data is copied
+    bool Send(std::string_view data, bool binary);
 
     void Close();
 

python-webrtc/cpp/src/interfaces/rtc_dtmf_sender.cpp

@@ -7,8 +7,6 @@
 
 #include "rtc_dtmf_sender.h"
 
-#include <algorithm>
-
 #include "../exceptions.h"
 #include "../utils/gil.h"
 #include "../utils/libwebrtc_thread.h"
@@ -93,11 +91,7 @@ namespace python_webrtc {
       const std::scoped_lock lock(_bufferMutex);
       ++_insertions;
     }
-    // libwebrtc takes longer tones and gaps than the specification
-    constexpr int minDuration = 70;
-    constexpr int minInterToneGap = 50;
-    if (!_dtmf->InsertDtmf(tones, std::max(duration, minDuration), std::max(interToneGap, minInterToneGap)) &&
-        !tones.empty()) {
+    if (!_dtmf->InsertDtmf(tones, duration, interToneGap) && !tones.empty()) {
       throw RTCException(webrtc::RTCErrorType::INVALID_STATE, "The DTMF sender can't send tones yet");
     }
     const std::scoped_lock lock(_bufferMutex);

python-webrtc/cpp/src/interfaces/rtc_ice_transport.cpp

@@ -8,36 +8,49 @@
 #include "rtc_ice_transport.h"
 
 #include <algorithm>
+#include <cstdint>
 
 #include <api/environment/environment_factory.h>
-#include <p2p/base/basic_packet_socket_factory.h>
 #include <p2p/base/p2p_constants.h>
 #include <p2p/base/p2p_transport_channel.h>
-#include <p2p/client/basic_port_allocator.h>
+#include <p2p/base/port_allocator.h>
 #include <pc/ice_server_parsing.h>
 #include <pc/ice_transport.h>
 #include <rtc_base/crypto_random.h>
-#include <rtc_base/network.h>
 
 #include <pybind11/stl.h>
 
 #include "../exceptions.h"
 #include "../utils/gil.h"
 #include "../utils/libwebrtc_thread.h"
+#include "../utils/port_allocator.h"
 
 namespace python_webrtc {
 
-  // Owned by the transport, created and destroyed on the network thread, after the ICE transport that uses it
-  struct StandaloneIce {
-    StandaloneIce(webrtc::Thread *networkThread, const PeerConnectionFactory &factory)
-        : env(webrtc::CreateEnvironment(factory.fieldTrials().CreateCopy())),
-          networkManager(std::make_unique<webrtc::BasicNetworkManager>(env, networkThread->socketserver())),
-          socketFactory(std::make_unique<webrtc::BasicPacketSocketFactory>(networkThread->socketserver())),
-          allocator(std::make_unique<webrtc::BasicPortAllocator>(env, networkManager.get(), socketFactory.get())) {
+  namespace {
+    // what libwebrtc gives a connection's allocator, on the network thread
+    std::unique_ptr<webrtc::PortAllocator> CreateStandalonePortAllocator(const webrtc::Environment &env,
+                                                                         PeerConnectionFactory &factory) {
+      auto allocator = CreatePortAllocator(env, factory.networkManager(), factory.packetSocketFactory());
       allocator->Initialize();
-      // like the connections of the factory
+      uint32_t flags =
+          allocator->flags() | webrtc::PORTALLOCATOR_ENABLE_SHARED_SOCKET | webrtc::PORTALLOCATOR_ENABLE_IPV6_ON_WIFI;
+      if (!env.field_trials().IsDisabled("WebRTC-IPv6Default")) {
+        flags |= webrtc::PORTALLOCATOR_ENABLE_IPV6;
+      }
+      allocator->set_flags(flags);
+      // like our connections: no waits between gathering phases
+      allocator->set_step_delay(0);
       allocator->SetNetworkIgnoreMask(factory.networkIgnoreMask());
+      return allocator;
     }
+  } // namespace
+
+  // Owned by the transport, created and destroyed on the network thread, after the ICE transport that uses it
+  struct StandaloneIce {
+    explicit StandaloneIce(PeerConnectionFactory &factory)
+        : env(webrtc::CreateEnvironment(factory.fieldTrials().CreateCopy())),
+          allocator(CreateStandalonePortAllocator(env, factory)) {}
 
     ~StandaloneIce() {
       if (transport) {
@@ -50,9 +63,7 @@ namespace python_webrtc {
     StandaloneIce &operator=(const StandaloneIce &) = delete;
 
     webrtc::Environment env;
-    std::unique_ptr<webrtc::BasicNetworkManager> networkManager;
-    std::unique_ptr<webrtc::BasicPacketSocketFactory> socketFactory;
-    std::unique_ptr<webrtc::BasicPortAllocator> allocator;
+    std::unique_ptr<webrtc::PortAllocator> allocator;
     std::unique_ptr<webrtc::P2PTransportChannel> channel;
     webrtc::scoped_refptr<webrtc::IceTransportWithPointer> transport;
   };
@@ -83,7 +94,7 @@ namespace python_webrtc {
         _subscribed->RemoveGatheringStateCallback(this);
       }
       if (_standalone) {
-        // on the network thread, before what it uses
+        // on the network thread, before the factory whose network manager it shares
         _transport = nullptr;
         _standalone = nullptr;
       }
@@ -422,7 +433,7 @@ namespace python_webrtc {
     std::pair<std::string, std::string> parameters(webrtc::CreateRandomString(webrtc::ICE_UFRAG_LENGTH),
                                                    webrtc::CreateRandomString(webrtc::ICE_PWD_LENGTH));
     BlockingCallOn(factory->workerThread(), [&]() {
-      standalone = std::make_shared<StandaloneIce>(factory->workerThread(), *factory);
+      standalone = std::make_shared<StandaloneIce>(*factory);
       webrtc::IceTransportInit init(standalone->env);
       init.set_port_allocator(standalone->allocator.get());
       standalone->channel = webrtc::P2PTransportChannel::Create("", 1, std::move(init));

python-webrtc/cpp/src/interfaces/rtc_peer_connection/descriptions.cpp

@@ -280,6 +280,12 @@ namespace python_webrtc {
       if (!self || self->IsClosed()) {
         completion->Fail(RTCCallbackException(closedError(remote ? "setRemoteDescription" : "setLocalDescription")));
       } else if (error.ok()) {
+        if (remote) {
+          // a new remote description may reuse the address of the one the end of candidates was for
+          const std::scoped_lock lock(self->_remoteEndOfCandidatesMutex);
+          self->_remoteEndOfCandidates.clear();
+          self->_remoteEndOfCandidatesDescription = nullptr;
+        }
         // the descriptions as the operation left them, before candidates gathered later
         self->_completionSnapshot = self->SnapshotDescriptions();
         auto created = self->WrapTransports();

python-webrtc/cpp/src/interfaces/rtc_peer_connection/rtc_peer_connection.cpp

@@ -32,6 +32,8 @@ namespace python_webrtc {
     auto configuration = webrtc::PeerConnectionInterface::RTCConfiguration();
     configuration.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan;
     configuration = _configuration.Apply(configuration);
+    // renders each decoded frame at once, without a smoothing queue per video receiver
+    configuration.set_prerenderer_smoothing(false);
 
     webrtc::PeerConnectionDependencies dependencies(this);
     dependencies.allocator = _factory->CreatePortAllocator(configuration);

python-webrtc/cpp/src/interfaces/rtc_rtp_sender.cpp

@@ -22,7 +22,7 @@ namespace python_webrtc {
 
   namespace {
 
-    // only what setParameters may change, as in Chromium
+    // only the encoding members setParameters may change, the rest keep their current values
     void applySettable(const webrtc::RtpEncodingParameters &requested, webrtc::RtpEncodingParameters &current) {
       current.active = requested.active;
       current.request_key_frame = requested.request_key_frame;