From 98badbcd9f69ec8434682eac4b96f2a4b5f27707 Mon Sep 17 00:00:00 2001 From: Emircan Uysaler Date: Thu, 28 Jun 2018 10:59:02 -0700 Subject: [PATCH] Add VP9 profile negotiation to SDP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL adds VP9 profile information in SDP. It adds the necessary fields and enums to codec containers. Additional profiles will be followed. Bug: webrtc:9376 Change-Id: I78574714f06f8087262a71dd64c01f31a229dd54 Reviewed-on: https://webrtc-review.googlesource.com/81960 Reviewed-by: Taylor (left Google) Reviewed-by: Niklas Enbom Reviewed-by: Rasmus Brandt Reviewed-by: Erik Språng Commit-Queue: Emircan Uysaler Cr-Commit-Position: refs/heads/master@{#23810} --- media/BUILD.gn | 16 +++++ media/base/codec.cc | 12 +++- media/base/codec_unittest.cc | 32 ++++++++++ media/base/vp9_profile.cc | 63 +++++++++++++++++++ media/base/vp9_profile.h | 53 ++++++++++++++++ media/engine/internaldecoderfactory.cc | 11 +--- media/engine/internalencoderfactory.cc | 9 +-- modules/video_coding/BUILD.gn | 3 + modules/video_coding/codecs/vp9/DEPS | 3 + modules/video_coding/codecs/vp9/include/vp9.h | 13 +++- modules/video_coding/codecs/vp9/vp9_impl.cc | 22 ++++--- modules/video_coding/codecs/vp9/vp9_impl.h | 5 +- modules/video_coding/codecs/vp9/vp9_noop.cc | 12 ++-- sdk/android/src/jni/vp9codec.cc | 4 +- 14 files changed, 223 insertions(+), 35 deletions(-) create mode 100644 media/base/vp9_profile.cc create mode 100644 media/base/vp9_profile.h create mode 100644 modules/video_coding/codecs/vp9/DEPS diff --git a/media/BUILD.gn b/media/BUILD.gn index f932a89549..03b82c645e 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -53,6 +53,20 @@ rtc_source_set("rtc_media_config") { ] } +rtc_source_set("rtc_vp9_profile") { + sources = [ + "base/vp9_profile.cc", + "base/vp9_profile.h", + ] + + deps = [ + "..:webrtc_common", + "../api/video_codecs:video_codecs_api", + "../rtc_base:rtc_base_approved", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + rtc_static_library("rtc_media_base") { visibility = [ "*" ] defines = [] @@ -109,6 +123,7 @@ rtc_static_library("rtc_media_base") { deps += [ ":rtc_h264_profile_id", ":rtc_media_config", + ":rtc_vp9_profile", "..:webrtc_common", "../api:libjingle_peerconnection_api", "../api/audio_codecs:audio_codecs_api", @@ -551,6 +566,7 @@ if (rtc_include_tests) { ":rtc_media", ":rtc_media_base", ":rtc_media_tests_utils", + ":rtc_vp9_profile", "../api:create_simulcast_test_fixture_api", "../api:libjingle_peerconnection_api", "../api:mock_video_codec_factory", diff --git a/media/base/codec.cc b/media/base/codec.cc index 7c1d67778a..caf3212112 100644 --- a/media/base/codec.cc +++ b/media/base/codec.cc @@ -13,6 +13,7 @@ #include #include "media/base/h264_profile_level_id.h" +#include "media/base/vp9_profile.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/stringencode.h" @@ -270,6 +271,8 @@ bool VideoCodec::Matches(const VideoCodec& other) const { if (CodecNamesEq(name.c_str(), kH264CodecName)) return webrtc::H264::IsSameH264Profile(params, other.params) && IsSameH264PacketizationMode(params, other.params); + if (CodecNamesEq(name.c_str(), kVp9CodecName)) + return webrtc::IsSameVP9Profile(params, other.params); return true; } @@ -386,9 +389,12 @@ bool IsSameCodec(const std::string& name1, // If different names (case insensitive), then not same formats. if (!CodecNamesEq(name1, name2)) return false; - // For every format besides H264, comparing names is enough. - return !CodecNamesEq(name1.c_str(), kH264CodecName) || - webrtc::H264::IsSameH264Profile(params1, params2); + // For every format besides H264 and VP9, comparing names is enough. + if (CodecNamesEq(name1.c_str(), kH264CodecName)) + return webrtc::H264::IsSameH264Profile(params1, params2); + if (CodecNamesEq(name1.c_str(), kVp9CodecName)) + return webrtc::IsSameVP9Profile(params1, params2); + return true; } } // namespace cricket diff --git a/media/base/codec_unittest.cc b/media/base/codec_unittest.cc index 36e0c3895c..54396a9244 100644 --- a/media/base/codec_unittest.cc +++ b/media/base/codec_unittest.cc @@ -9,6 +9,8 @@ */ #include "media/base/codec.h" + +#include "media/base/vp9_profile.h" #include "rtc_base/gunit.h" using cricket::AudioCodec; @@ -186,6 +188,36 @@ TEST(CodecTest, TestVideoCodecMatches) { EXPECT_FALSE(c1.Matches(VideoCodec(95, "V"))); } +// VP9 codecs compare profile information. +TEST(CodecTest, TestVP9CodecMatches) { + const char kProfile0[] = "0"; + const char kProfile2[] = "2"; + + VideoCodec c_no_profile(95, cricket::kVp9CodecName); + VideoCodec c_profile0(95, cricket::kVp9CodecName); + c_profile0.params[webrtc::kVP9FmtpProfileId] = kProfile0; + + EXPECT_TRUE(c_profile0.Matches(c_no_profile)); + + { + VideoCodec c_profile0_eq(95, cricket::kVp9CodecName); + c_profile0_eq.params[webrtc::kVP9FmtpProfileId] = kProfile0; + EXPECT_TRUE(c_profile0.Matches(c_profile0_eq)); + } + + { + VideoCodec c_profile2(95, cricket::kVp9CodecName); + c_profile2.params[webrtc::kVP9FmtpProfileId] = kProfile2; + EXPECT_FALSE(c_profile0.Matches(c_profile2)); + EXPECT_FALSE(c_no_profile.Matches(c_profile2)); + } + + { + VideoCodec c_no_profile_eq(95, cricket::kVp9CodecName); + EXPECT_TRUE(c_no_profile.Matches(c_no_profile_eq)); + } +} + // Matching H264 codecs also need to have matching profile-level-id and // packetization-mode. TEST(CodecTest, TestH264CodecMatches) { diff --git a/media/base/vp9_profile.cc b/media/base/vp9_profile.cc new file mode 100644 index 0000000000..3ced430e5f --- /dev/null +++ b/media/base/vp9_profile.cc @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/base/vp9_profile.h" + +#include "rtc_base/string_to_number.h" + +namespace webrtc { + +// Profile information for VP9 video. +const char kVP9FmtpProfileId[] = "x-google-profile-id"; + +std::string VP9ProfileToString(VP9Profile profile) { + switch (profile) { + case VP9Profile::kProfile0: + return "0"; + case VP9Profile::kProfile2: + return "2"; + } + return "0"; +} + +absl::optional StringToVP9Profile(const std::string& str) { + const absl::optional i = rtc::StringToNumber(str); + if (!i.has_value()) + return absl::nullopt; + + switch (i.value()) { + case 0: + return VP9Profile::kProfile0; + case 2: + return VP9Profile::kProfile2; + default: + return absl::nullopt; + } + return absl::nullopt; +} + +absl::optional ParseSdpForVP9Profile( + const SdpVideoFormat::Parameters& params) { + const auto profile_it = params.find(kVP9FmtpProfileId); + if (profile_it == params.end()) + return VP9Profile::kProfile0; + const std::string& profile_str = profile_it->second; + return StringToVP9Profile(profile_str); +} + +bool IsSameVP9Profile(const SdpVideoFormat::Parameters& params1, + const SdpVideoFormat::Parameters& params2) { + const rtc::Optional profile = ParseSdpForVP9Profile(params1); + const rtc::Optional other_profile = + ParseSdpForVP9Profile(params2); + return profile && other_profile && profile == other_profile; +} + +} // namespace webrtc diff --git a/media/base/vp9_profile.h b/media/base/vp9_profile.h new file mode 100644 index 0000000000..5cc1d166a8 --- /dev/null +++ b/media/base/vp9_profile.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_VP9_PROFILE_H_ +#define MEDIA_BASE_VP9_PROFILE_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "api/video_codecs/sdp_video_format.h" +#include "common_types.h" // NOLINT(build/include) + +namespace webrtc { + +// Profile information for VP9 video. +extern const char kVP9FmtpProfileId[]; + +enum class VP9Profile { + kProfile0, + kProfile2, +}; + +// Helper functions to convert VP9Profile to std::string. Returns "0" by +// default. +std::string VP9ProfileToString(VP9Profile profile); + +// Helper functions to convert std::string to VP9Profile. Returns null if given +// an invalid profile string. +absl::optional StringToVP9Profile(const std::string& str); + +// Parse profile that is represented as a string of single digit contained in an +// SDP key-value map. A default profile(kProfile0) will be returned if the +// profile key is missing. Nothing will be returned if the key is present but +// the string is invalid. +absl::optional ParseSdpForVP9Profile( + const SdpVideoFormat::Parameters& params); + +// Returns true if the parameters have the same VP9 profile, or neither contains +// VP9 profile. +bool IsSameVP9Profile(const SdpVideoFormat::Parameters& params1, + const SdpVideoFormat::Parameters& params2); + +} // namespace webrtc + +#endif // MEDIA_BASE_VP9_PROFILE_H_ diff --git a/media/engine/internaldecoderfactory.cc b/media/engine/internaldecoderfactory.cc index c9cd861f6b..08e29cfa6a 100644 --- a/media/engine/internaldecoderfactory.cc +++ b/media/engine/internaldecoderfactory.cc @@ -24,8 +24,8 @@ std::vector InternalDecoderFactory::GetSupportedFormats() const { std::vector formats; formats.push_back(SdpVideoFormat(cricket::kVp8CodecName)); - if (VP9Decoder::IsSupported()) - formats.push_back(SdpVideoFormat(cricket::kVp9CodecName)); + for (const SdpVideoFormat& format : SupportedVP9Codecs()) + formats.push_back(format); for (const SdpVideoFormat& h264_format : SupportedH264Codecs()) formats.push_back(h264_format); return formats; @@ -35,15 +35,10 @@ std::unique_ptr InternalDecoderFactory::CreateVideoDecoder( const SdpVideoFormat& format) { if (cricket::CodecNamesEq(format.name, cricket::kVp8CodecName)) return VP8Decoder::Create(); - - if (cricket::CodecNamesEq(format.name, cricket::kVp9CodecName)) { - RTC_DCHECK(VP9Decoder::IsSupported()); + if (cricket::CodecNamesEq(format.name, cricket::kVp9CodecName)) return VP9Decoder::Create(); - } - if (cricket::CodecNamesEq(format.name, cricket::kH264CodecName)) return H264Decoder::Create(); - RTC_LOG(LS_ERROR) << "Trying to create decoder for unsupported format"; return nullptr; } diff --git a/media/engine/internalencoderfactory.cc b/media/engine/internalencoderfactory.cc index 2f7df9f58f..f980c7dd03 100644 --- a/media/engine/internalencoderfactory.cc +++ b/media/engine/internalencoderfactory.cc @@ -24,12 +24,10 @@ std::vector InternalEncoderFactory::GetSupportedFormats() const { std::vector supported_codecs; supported_codecs.push_back(SdpVideoFormat(cricket::kVp8CodecName)); - if (webrtc::VP9Encoder::IsSupported()) - supported_codecs.push_back(SdpVideoFormat(cricket::kVp9CodecName)); - + for (const webrtc::SdpVideoFormat& format : webrtc::SupportedVP9Codecs()) + supported_codecs.push_back(format); for (const webrtc::SdpVideoFormat& format : webrtc::SupportedH264Codecs()) supported_codecs.push_back(format); - return supported_codecs; } @@ -45,13 +43,10 @@ std::unique_ptr InternalEncoderFactory::CreateVideoEncoder( const SdpVideoFormat& format) { if (cricket::CodecNamesEq(format.name, cricket::kVp8CodecName)) return VP8Encoder::Create(); - if (cricket::CodecNamesEq(format.name, cricket::kVp9CodecName)) return VP9Encoder::Create(); - if (cricket::CodecNamesEq(format.name, cricket::kH264CodecName)) return H264Encoder::Create(cricket::VideoCodec(format)); - RTC_LOG(LS_ERROR) << "Trying to created encoder of unsupported format " << format.name; return nullptr; diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index bff40c9620..68d4426705 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -472,7 +472,10 @@ rtc_static_library("webrtc_vp9") { ":webrtc_vp9_helpers", "..:module_api", "../..:webrtc_common", + "../../api/video_codecs:video_codecs_api", "../../common_video", + "../../media:rtc_media_base", + "../../media:rtc_vp9_profile", "../../rtc_base:checks", "../../rtc_base:rtc_base", "../../system_wrappers", diff --git a/modules/video_coding/codecs/vp9/DEPS b/modules/video_coding/codecs/vp9/DEPS new file mode 100644 index 0000000000..cc5cd70142 --- /dev/null +++ b/modules/video_coding/codecs/vp9/DEPS @@ -0,0 +1,3 @@ +include_rules = [ + "+media/base", +] diff --git a/modules/video_coding/codecs/vp9/include/vp9.h b/modules/video_coding/codecs/vp9/include/vp9.h index 39dc138b23..8091cacec9 100644 --- a/modules/video_coding/codecs/vp9/include/vp9.h +++ b/modules/video_coding/codecs/vp9/include/vp9.h @@ -13,22 +13,31 @@ #define MODULES_VIDEO_CODING_CODECS_VP9_INCLUDE_VP9_H_ #include +#include +#include "api/video_codecs/sdp_video_format.h" +#include "media/base/codec.h" #include "modules/video_coding/include/video_codec_interface.h" namespace webrtc { +// Returns a vector with all supported internal VP9 profiles that we can +// negotiate in SDP, in order of preference. +std::vector SupportedVP9Codecs(); + class VP9Encoder : public VideoEncoder { public: - static bool IsSupported(); + // Deprecated. Returns default implementation using VP9 Profile 0. + // TODO(emircan): Remove once this is no longer used. static std::unique_ptr Create(); + // Parses VP9 Profile from |codec| and returns the appropriate implementation. + static std::unique_ptr Create(const cricket::VideoCodec& codec); ~VP9Encoder() override {} }; class VP9Decoder : public VideoDecoder { public: - static bool IsSupported(); static std::unique_ptr Create(); ~VP9Decoder() override {} diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc index 5b7f5b95c5..39ab7b3206 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -52,12 +52,19 @@ int GetCpuSpeed(int width, int height) { #endif } -bool VP9Encoder::IsSupported() { - return true; +std::vector SupportedVP9Codecs() { + return {SdpVideoFormat( + cricket::kVp9CodecName, + {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile0)}})}; } std::unique_ptr VP9Encoder::Create() { - return rtc::MakeUnique(); + return rtc::MakeUnique(cricket::VideoCodec()); +} + +std::unique_ptr VP9Encoder::Create( + const cricket::VideoCodec& codec) { + return rtc::MakeUnique(codec); } void VP9EncoderImpl::EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt* pkt, @@ -66,9 +73,11 @@ void VP9EncoderImpl::EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt* pkt, enc->GetEncodedLayerFrame(pkt); } -VP9EncoderImpl::VP9EncoderImpl() +VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec) : encoded_image_(), encoded_complete_callback_(nullptr), + profile_( + ParseSdpForVP9Profile(codec.params).value_or(VP9Profile::kProfile0)), inited_(false), timestamp_(0), cpu_speed_(3), @@ -88,6 +97,7 @@ VP9EncoderImpl::VP9EncoderImpl() is_flexible_mode_(false) { memset(&codec_, 0, sizeof(codec_)); memset(&svc_params_, 0, sizeof(vpx_svc_extra_cfg_t)); + RTC_DCHECK(VP9Profile::kProfile0 == profile_); } VP9EncoderImpl::~VP9EncoderImpl() { @@ -960,10 +970,6 @@ const char* VP9EncoderImpl::ImplementationName() const { return "libvpx"; } -bool VP9Decoder::IsSupported() { - return true; -} - std::unique_ptr VP9Decoder::Create() { return rtc::MakeUnique(); } diff --git a/modules/video_coding/codecs/vp9/vp9_impl.h b/modules/video_coding/codecs/vp9/vp9_impl.h index 5cf09a3645..7009311072 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/modules/video_coding/codecs/vp9/vp9_impl.h @@ -17,6 +17,8 @@ #include #include "modules/video_coding/codecs/vp9/include/vp9.h" + +#include "media/base/vp9_profile.h" #include "modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h" #include "rtc_base/rate_statistics.h" @@ -28,7 +30,7 @@ namespace webrtc { class VP9EncoderImpl : public VP9Encoder { public: - VP9EncoderImpl(); + explicit VP9EncoderImpl(const cricket::VideoCodec& codec); virtual ~VP9EncoderImpl(); @@ -94,6 +96,7 @@ class VP9EncoderImpl : public VP9Encoder { CodecSpecificInfo codec_specific_; EncodedImageCallback* encoded_complete_callback_; VideoCodec codec_; + const VP9Profile profile_; bool inited_; int64_t timestamp_; int cpu_speed_; diff --git a/modules/video_coding/codecs/vp9/vp9_noop.cc b/modules/video_coding/codecs/vp9/vp9_noop.cc index baa04862c3..b058c2d672 100644 --- a/modules/video_coding/codecs/vp9/vp9_noop.cc +++ b/modules/video_coding/codecs/vp9/vp9_noop.cc @@ -14,12 +14,14 @@ #endif // !defined(RTC_DISABLE_VP9) #include "modules/video_coding/codecs/vp9/include/vp9.h" + +#include "api/video_codecs/sdp_video_format.h" #include "rtc_base/checks.h" namespace webrtc { -bool VP9Encoder::IsSupported() { - return false; +std::vector SupportedVP9Codecs() { + return std::vector(); } std::unique_ptr VP9Encoder::Create() { @@ -27,8 +29,10 @@ std::unique_ptr VP9Encoder::Create() { return nullptr; } -bool VP9Decoder::IsSupported() { - return false; +std::unique_ptr VP9Encoder::Create( + const cricket::VideoCodec& codec) { + RTC_NOTREACHED(); + return nullptr; } std::unique_ptr VP9Decoder::Create() { diff --git a/sdk/android/src/jni/vp9codec.cc b/sdk/android/src/jni/vp9codec.cc index d9cc387016..ca712e6d86 100644 --- a/sdk/android/src/jni/vp9codec.cc +++ b/sdk/android/src/jni/vp9codec.cc @@ -25,7 +25,7 @@ static jlong JNI_VP9Encoder_CreateEncoder(JNIEnv* jni, static jboolean JNI_VP9Encoder_IsSupported(JNIEnv* jni, const JavaParamRef&) { - return VP9Encoder::IsSupported(); + return !SupportedVP9Codecs().empty(); } static jlong JNI_VP9Decoder_CreateDecoder(JNIEnv* jni, @@ -35,7 +35,7 @@ static jlong JNI_VP9Decoder_CreateDecoder(JNIEnv* jni, static jboolean JNI_VP9Decoder_IsSupported(JNIEnv* jni, const JavaParamRef&) { - return VP9Decoder::IsSupported(); + return !SupportedVP9Codecs().empty(); } } // namespace jni