fix: make decoder QP parsing flag atomic and give up codec calls without a JNI frame

webrtc-jni/src/main/cpp/include/media/video/codec/JavaLocalFrame.h

@@ -25,18 +25,27 @@ namespace jni
 	// thread stays attached to the JVM for as long as it runs, so without a
 	// frame of their own the references made for one call would pile up
 	// behind those of the next.
+	//
+	// A frame that cannot be pushed reports the JVM's exception as a codec
+	// exception, and the call it was meant for gives up rather than make
+	// references that nothing would free soon: see pushed().
 	class JavaLocalFrame
 	{
 		public:
-			JavaLocalFrame(JNIEnv * env, jint capacity);
+			// The context names the call in the log.
+			JavaLocalFrame(JNIEnv * env, jint capacity, const char * context);
 			~JavaLocalFrame();
 
 			JavaLocalFrame(const JavaLocalFrame &) = delete;
 			JavaLocalFrame & operator=(const JavaLocalFrame &) = delete;
 
+			// Whether the frame is in place, which a call checks before it
+			// makes any local reference.
+			bool pushed() const;
+
 		private:
 			JNIEnv * env;
-			bool pushed;
+			const bool isPushed;
 	};
 }
 

webrtc-jni/src/main/cpp/include/media/video/codec/VideoDecoderWrapper.h

@@ -117,8 +117,10 @@ namespace jni
 			Settings settings;
 			bool initialized;
 
-			// Parsing stops while the decoder provides the QP itself.
-			bool qpParsingEnabled;
+			// Parsing stops while the decoder provides the QP itself. Read on
+			// the decoder thread and written on the Java decoder's, which
+			// may deliver its frames from a thread of its own.
+			std::atomic<bool> qpParsingEnabled;
 			webrtc::H264BitstreamParser h264BitstreamParser;
 	};
 }

webrtc-jni/src/main/cpp/src/media/video/codec/JavaLocalFrame.cpp

@@ -15,23 +15,34 @@
  */
 
 #include "media/video/codec/JavaLocalFrame.h"
+#include "media/video/codec/VideoCodecUtils.h"
+
+#include "rtc_base/logging.h"
 
 namespace jni
 {
-	JavaLocalFrame::JavaLocalFrame(JNIEnv * env, jint capacity) :
+	JavaLocalFrame::JavaLocalFrame(JNIEnv * env, jint capacity, const char * context) :
 		env(env),
-		pushed(env->PushLocalFrame(capacity) == JNI_OK)
+		isPushed(env->PushLocalFrame(capacity) == JNI_OK)
 	{
-		if (!pushed) {
-			// Out of memory: the references then go to the enclosing frame.
-			env->ExceptionClear();
+		if (!isPushed) {
+			// The JVM is out of memory and has thrown, which is reported like
+			// any exception of a codec call.
+			ClearCodecException(env, context);
+
+			RTC_LOG(LS_WARNING) << "No local JNI frame for " << context << ", giving up the call";
 		}
 	}
 
 	JavaLocalFrame::~JavaLocalFrame()
 	{
-		if (pushed) {
+		if (isPushed) {
 			env->PopLocalFrame(nullptr);
 		}
 	}
+
+	bool JavaLocalFrame::pushed() const
+	{
+		return isPushed;
+	}
 }

webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderFactoryWrapper.cpp

@@ -53,7 +53,11 @@ namespace jni
 			return nullptr;
 		}
 
-		JavaLocalFrame localFrame(env, 16);
+		JavaLocalFrame localFrame(env, 16, "createDecoder");
+
+		if (!localFrame.pushed()) {
+			return nullptr;
+		}
 
 		try {
 			JavaLocalRef<jobject> info = VideoCodecInfo::toJava(env, format);

webrtc-jni/src/main/cpp/src/media/video/codec/VideoDecoderWrapper.cpp

@@ -57,7 +57,12 @@ namespace jni
 		initialized(false),
 		qpParsingEnabled(true)
 	{
-		JavaLocalFrame localFrame(env, kLocalFrameCapacity);
+		JavaLocalFrame localFrame(env, kLocalFrameCapacity, "getImplementationName");
+
+		if (!localFrame.pushed()) {
+			// A decoder without a name, taken for a software one.
+			return;
+		}
 
 		jobject name = env->CallObjectMethod(decoder, javaClass->getImplementationName);
 
@@ -102,7 +107,16 @@ namespace jni
 
 	bool VideoDecoderWrapper::ConfigureInternal(JNIEnv * env)
 	{
-		JavaLocalFrame localFrame(env, kLocalFrameCapacity);
+		JavaLocalFrame localFrame(env, kLocalFrameCapacity, "initDecode");
+
+		if (!localFrame.pushed()) {
+			// Detaching the earlier callback makes no local reference.
+			InvalidateCallback(env);
+
+			initialized = false;
+
+			return false;
+		}
 
 		webrtc::RenderResolution resolution = settings.max_render_resolution();
 
@@ -154,7 +168,13 @@ namespace jni
 			return WEBRTC_VIDEO_CODEC_ERROR;
 		}
 
-		JavaLocalFrame localFrame(env, kLocalFrameCapacity);
+		JavaLocalFrame localFrame(env, kLocalFrameCapacity, "decode");
+
+		if (!localFrame.pushed()) {
+			// Given up before a record of the image is made, so the decoder
+			// is not waited for.
+			return WEBRTC_VIDEO_CODEC_ERROR;
+		}
 
 		// The capture time identifies the frame, and received images carry
 		// none, so it is derived from the RTP timestamp.
@@ -215,11 +235,18 @@ namespace jni
 
 	int32_t VideoDecoderWrapper::ReleaseInternal(JNIEnv * env)
 	{
-		JavaLocalFrame localFrame(env, kLocalFrameCapacity);
+		// Releasing goes ahead without a frame, since giving it up would keep
+		// the Java decoder's resources, and the status is the only reference
+		// it makes.
+		JavaLocalFrame localFrame(env, kLocalFrameCapacity, "release");
 
 		jobject status = env->CallObjectMethod(decoder, javaClass->release);
 		int32_t result = ToNativeCodecStatus(env, status, "release");
 
+		if (!localFrame.pushed() && status != nullptr) {
+			env->DeleteLocalRef(status);
+		}
+
 		RTC_LOG(LS_INFO) << "Java decoder release: " << result;
 
 		InvalidateCallback(env);

webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderFactoryWrapper.cpp

@@ -53,7 +53,11 @@ namespace jni
 			return nullptr;
 		}
 
-		JavaLocalFrame localFrame(env, 16);
+		JavaLocalFrame localFrame(env, 16, "createEncoder");
+
+		if (!localFrame.pushed()) {
+			return nullptr;
+		}
 
 		try {
 			JavaLocalRef<jobject> info = VideoCodecInfo::toJava(env, format);

webrtc-jni/src/main/cpp/src/media/video/codec/VideoEncoderWrapper.cpp

@@ -106,7 +106,16 @@ namespace jni
 
 	int32_t VideoEncoderWrapper::InitEncodeInternal(JNIEnv * env)
 	{
-		JavaLocalFrame localFrame(env, kLocalFrameCapacity);
+		JavaLocalFrame localFrame(env, kLocalFrameCapacity, "initEncode");
+
+		if (!localFrame.pushed()) {
+			// Detaching the earlier callback makes no local reference.
+			InvalidateCallback(env);
+
+			initialized = false;
+
+			return WEBRTC_VIDEO_CODEC_ERROR;
+		}
 
 		bool automaticResizeOn;
 
@@ -192,11 +201,18 @@ namespace jni
 
 	int32_t VideoEncoderWrapper::ReleaseInternal(JNIEnv * env)
 	{
-		JavaLocalFrame localFrame(env, kLocalFrameCapacity);
+		// Releasing goes ahead without a frame, since giving it up would keep
+		// the Java encoder's resources, and the status is the only reference
+		// it makes.
+		JavaLocalFrame localFrame(env, kLocalFrameCapacity, "release");
 
 		jobject status = env->CallObjectMethod(encoder, javaClass->release);
 		int32_t result = ToNativeCodecStatus(env, status, "release");
 
+		if (!localFrame.pushed() && status != nullptr) {
+			env->DeleteLocalRef(status);
+		}
+
 		RTC_LOG(LS_INFO) << "Java encoder release: " << result;
 
 		InvalidateCallback(env);
@@ -224,7 +240,15 @@ namespace jni
 			return WEBRTC_VIDEO_CODEC_ERROR;
 		}
 
-		JavaLocalFrame localFrame(env, kLocalFrameCapacity);
+		JavaLocalFrame localFrame(env, kLocalFrameCapacity, "encode");
+
+		if (!localFrame.pushed()) {
+			// Dropped rather than failed, since WebRTC gives up an encoder that
+			// fails, and running short of memory is no fault of the encoder.
+			ReportDroppedFrame(frame.rtp_timestamp());
+
+			return WEBRTC_VIDEO_CODEC_OK;
+		}
 
 		const auto javaFrameClass = JavaClasses::get<JavaVideoFrameClass>(env);
 		jobject jFrame = nullptr;
@@ -309,7 +333,12 @@ namespace jni
 			return;
 		}
 
-		JavaLocalFrame localFrame(env, kLocalFrameCapacity);
+		JavaLocalFrame localFrame(env, kLocalFrameCapacity, "setRates");
+
+		if (!localFrame.pushed()) {
+			// The encoder keeps its rates until the next call.
+			return;
+		}
 
 		jobjectArray layers = env->NewObjectArray(webrtc::kMaxSpatialLayers, javaClass->intArrayClass, nullptr);
 
@@ -465,7 +494,12 @@ namespace jni
 
 	void VideoEncoderWrapper::UpdateEncoderInfo(JNIEnv * env)
 	{
-		JavaLocalFrame localFrame(env, kLocalFrameCapacity);
+		JavaLocalFrame localFrame(env, kLocalFrameCapacity, "getEncoderInfo");
+
+		if (!localFrame.pushed()) {
+			// The info stays as it was.
+			return;
+		}
 
 		encoderInfo.supports_native_handle = false;
 		encoderInfo.scaling_settings = GetScalingSettings(env);