上图是之前 Android-RTC-1 简单分析得出来的,一张关于PeerConnectionFactory的至下而上结构图。本篇文章Android-RTC-5,继续拆分createPeerConnectionFactory当中关于音频模块的构造部分。
在java层的createPeerConnectionFactory入口函数当中,还有三个和音频相关的Factory,AudioEncoderFactoryFactory / AudioDecoderFactoryFactory / AudioProcessingFactory
//org.webrtc.PeerConnectionFactory.Builder
private AudioEncoderFactoryFactory audioEncoderFactoryFactory =
new BuiltinAudioEncoderFactoryFactory();
private AudioDecoderFactoryFactory audioDecoderFactoryFactory =
new BuiltinAudioDecoderFactoryFactory();
public PeerConnectionFactory createPeerConnectionFactory() {
PeerConnectionFactory.checkInitializeHasBeenCalled();
if (this.audioDeviceModule == null) {
this.audioDeviceModule = JavaAudioDeviceModule.builder(ContextUtils.getApplicationContext()).createAudioDeviceModule();
}
return PeerConnectionFactory.nativeCreatePeerConnectionFactory(
ContextUtils.getApplicationContext(),
this.options,
this.audioDeviceModule.getNativeAudioDeviceModulePointer(),
this.audioEncoderFactoryFactory.createNativeAudioEncoderFactory(),
this.audioDecoderFactoryFactory.createNativeAudioDecoderFactory(),
this.videoEncoderFactory,
this.videoDecoderFactory,
this.audioProcessingFactory == null ? 0L : this.audioProcessingFactory.createNative(),
this.fecControllerFactoryFactory == null ? 0L : this.fecControllerFactoryFactory.createNative(),
this.networkControllerFactoryFactory == null ? 0L : this.networkControllerFactoryFactory.createNativeNetworkControllerFactory(),
this.networkStatePredictorFactoryFactory == null ? 0L : this.networkStatePredictorFactoryFactory.createNativeNetworkStatePredictorFactory(),
this.neteqFactoryFactory == null ? 0L : this.neteqFactoryFactory.createNativeNetEqFactory()
);
}
一、AudioEncoder/DecoderFactory
第一眼看AudioEncoderFactoryFactory ,好家伙!Factory的Factory。深入了解发现这只是个实现createFactory接口的类命名而已,别被外表吓唬到了。
/**
* Implementations of this interface can create a native {@code webrtc::AudioDecoderFactory}.
*/
public interface AudioDecoderFactoryFactory {
/**
* Returns a pointer to a {@code webrtc::AudioDecoderFactory}. The caller takes ownership.
*/
long createNativeAudioDecoderFactory();
}
/**
* Creates a native {@code webrtc::AudioDecoderFactory} with the builtin audio decoders.
*/
public class BuiltinAudioDecoderFactoryFactory implements AudioDecoderFactoryFactory {
@Override
public long createNativeAudioDecoderFactory() {
return nativeCreateBuiltinAudioDecoderFactory();
}
private static native long nativeCreateBuiltinAudioDecoderFactory();
}
接下来我们以Encoder为例,以nativeCreateBuiltinAudioDecoderFactory进入看看AudioFactory的模块构造。
// out\arm64-v8a\gen\sdk\android\generated_builtin_audio_codecs_jni\BuiltinAudioEncoderFactoryFactory_jni.h
namespace webrtc {
namespace jni {
static jlong JNI_BuiltinAudioEncoderFactoryFactory_CreateBuiltinAudioEncoderFactory(JNIEnv* env);
JNI_GENERATOR_EXPORT jlong
Java_org_webrtc_BuiltinAudioEncoderFactoryFactory_nativeCreateBuiltinAudioEncoderFactory(
JNIEnv* env,
jclass jcaller) {
return JNI_BuiltinAudioEncoderFactoryFactory_CreateBuiltinAudioEncoderFactory(env);
}
} // namespace jni
} // namespace webrtc
// sdk\android\src\jni\builtin_audio_encoder_factory_factory.cc
static jlong
JNI_BuiltinAudioEncoderFactoryFactory_CreateBuiltinAudioEncoderFactory(
JNIEnv* env) {
return NativeToJavaPointer(CreateBuiltinAudioEncoderFactory().release());
// NativeToJavaPointer 和 上一篇的 jlongFromPointer差不多。
// 都是把对象指针转化long值保存到java对象,类似于对象绑定的处理。
}
// api\audio_codecs\builtin_audio_encoder_factory.cc
rtc::scoped_refptr<AudioEncoderFactory> CreateBuiltinAudioEncoderFactory() {
return CreateAudioEncoderFactory<
#if WEBRTC_USE_BUILTIN_OPUS
AudioEncoderOpus, NotAdvertised<AudioEncoderMultiChannelOpus>,
#endif
AudioEncoderIsac, AudioEncoderG722,
#if WEBRTC_USE_BUILTIN_ILBC
AudioEncoderIlbc,
#endif
AudioEncoderG711, NotAdvertised<AudioEncoderL16>>();
}
函数CreateBuiltinAudioEncoderFactory的内容一眼看有点吓唬人,CreateAudioEncoderFactory这个动态模板类有点厉害,没有深厚的C++功底不敢写or写不出这样的代码。
// api\audio_codecs\audio_encoder_factory_template.h
template <typename... Ts>
rtc::scoped_refptr<AudioEncoderFactory> CreateAudioEncoderFactory() {
// There's no technical reason we couldn't allow zero template parameters,
// but such a factory couldn't create any encoders, and callers can do this
// by mistake by simply forgetting the <> altogether. So we forbid it in
// order to prevent caller foot-shooting.
static_assert(sizeof...(Ts) >= 1,
"Caller must give at least one template parameter");
return rtc::make_ref_counted<
audio_encoder_factory_template_impl::AudioEncoderFactoryT<Ts...>>();
}
template <typename... Ts>
class AudioEncoderFactoryT : public AudioEncoderFactory {
public:
std::vector<AudioCodecSpec> GetSupportedEncoders() override {
std::vector<AudioCodecSpec> specs;
Helper<Ts...>::AppendSupportedEncoders(&specs);
return specs;
}
absl::optional<AudioCodecInfo> QueryAudioEncoder(
const SdpAudioFormat& format) override {
return Helper<Ts...>::QueryAudioEncoder(format);
}
std::unique_ptr<AudioEncoder> MakeAudioEncoder(
int payload_type,
const SdpAudioFormat& format,
absl::optional<AudioCodecPairId> codec_pair_id) override {
return Helper<Ts...>::MakeAudioEncoder(payload_type, format, codec_pair_id);
}
};
深入看看这个AudioEncoderFactory的创建,可以看到Helper<Ts...>是关键。我们以MakeAudioEncoder为例,跟踪Helper。
namespace audio_encoder_factory_template_impl {
template <typename... Ts>
struct Helper;
// Base case: 0 template parameters.
template <>
struct Helper<> {
static void AppendSupportedEncoders(std::vector<AudioCodecSpec>* specs) {}
static absl::optional<AudioCodecInfo> QueryAudioEncoder(
const SdpAudioFormat& format) {
return absl::nullopt;
}
static std::unique_ptr<AudioEncoder> MakeAudioEncoder(
int payload_type,
const SdpAudioFormat& format,
absl::optional<AudioCodecPairId> codec_pair_id) {
return nullptr;
}
};
// Inductive case: Called with n + 1 template parameters; calls subroutines
// with n template parameters.
template <typename T, typename... Ts>
struct Helper<T, Ts...> {
static void AppendSupportedEncoders(std::vector<AudioCodecSpec>* specs) {
T::AppendSupportedEncoders(specs);
Helper<Ts...>::AppendSupportedEncoders(specs);
}
static absl::optional<AudioCodecInfo> QueryAudioEncoder(
const SdpAudioFormat& format) {
auto opt_config = T::SdpToConfig(format);
static_assert(std::is_same<decltype(opt_config),
absl::optional<typename T::Config>>::value,
"T::SdpToConfig() must return a value of type "
"absl::optional<T::Config>");
return opt_config ? absl::optional<AudioCodecInfo>(
T::QueryAudioEncoder(*opt_config))
: Helper<Ts...>::QueryAudioEncoder(format);
}
static std::unique_ptr<AudioEncoder> MakeAudioEncoder(
int payload_type,
const SdpAudioFormat& format,
absl::optional<AudioCodecPairId> codec_pair_id) {
auto opt_config = T::SdpToConfig(format);
if (opt_config) {
return T::MakeAudioEncoder(*opt_config, payload_type, codec_pair_id);
} else {
return Helper<Ts...>::MakeAudioEncoder(payload_type, format,
codec_pair_id);
}
}
};
template <typename... Ts>
class AudioEncoderFactoryT : public AudioEncoderFactory { ... }
} // namespace audio_encoder_factory_template_impl
代码显示比较明了,就是化简为零的递归思路,按照动态输入的AudioEncoder一个个往后传。直到最后一个AudioEncoder为止,。这里还能清楚的看到,AudioEncoderFactory只有3个功能函数,GetSupportedEncoders / QueryAudioEncoder / MakeAudioEncoder,这些功能函数,通过Helper间接调用AudioEncoder的相应实现,以到达适配正确的AudioEncoder。
// api\audio_codecs\g711\audio_encoder_g711.h
struct RTC_EXPORT AudioEncoderG711 {
struct Config {
enum class Type { kPcmU, kPcmA };
bool IsOk() const {
return (type == Type::kPcmU || type == Type::kPcmA) &&
frame_size_ms > 0 && frame_size_ms % 10 == 0 && num_channels >= 1;
}
Type type = Type::kPcmU;
int num_channels = 1;
int frame_size_ms = 20;
};
static absl::optional<AudioEncoderG711::Config> SdpToConfig(
const SdpAudioFormat& audio_format);
static void AppendSupportedEncoders(std::vector<AudioCodecSpec>* specs);
static AudioCodecInfo QueryAudioEncoder(const Config& config);
static std::unique_ptr<AudioEncoder> MakeAudioEncoder(
const Config& config,
int payload_type,
absl::optional<AudioCodecPairId> codec_pair_id = absl::nullopt);
};
// api\audio_codecs\g711\audio_encoder_g711.cc
absl::optional<AudioEncoderG711::Config> AudioEncoderG711::SdpToConfig(
const SdpAudioFormat& format) {
const bool is_pcmu = absl::EqualsIgnoreCase(format.name, "PCMU");
const bool is_pcma = absl::EqualsIgnoreCase(format.name, "PCMA");
if (format.clockrate_hz == 8000 && format.num_channels >= 1 &&
(is_pcmu || is_pcma)) {
Config config;
config.type = is_pcmu ? Config::Type::kPcmU : Config::Type::kPcmA;
config.num_channels = rtc::dchecked_cast<int>(format.num_channels);
config.frame_size_ms = 20;
auto ptime_iter = format.parameters.find("ptime");
if (ptime_iter != format.parameters.end()) {
const auto ptime = rtc::StringToNumber<int>(ptime_iter->second);
if (ptime && *ptime > 0) {
config.frame_size_ms = rtc::SafeClamp(10 * (*ptime / 10), 10, 60);
}
}
RTC_DCHECK(config.IsOk());
return config;
} else {
return absl::nullopt;
}
}
void AudioEncoderG711::AppendSupportedEncoders(
std::vector<AudioCodecSpec>* specs) {
for (const char* type : {"PCMU", "PCMA"}) {
specs->push_back({
{type, 8000, 1}, {8000, 1, 64000}});
}
}
AudioCodecInfo AudioEncoderG711::QueryAudioEncoder(const Config& config) {
RTC_DCHECK(config.IsOk());
return {8000, rtc::dchecked_cast<size_t>(config.num_channels),
64000 * config.num_channels};
}
std::unique_ptr<AudioEncoder> AudioEncoderG711::MakeAudioEncoder(
const Config& config,
int payload_type,
absl::optional<AudioCodecPairId> /*codec_pair_id*/) {
RTC_DCHECK(config.IsOk());
switch (config.type) {
case Config::Type::kPcmU: {
AudioEncoderPcmU::Config impl_config;
impl_config.num_channels = config.num_channels;
impl_config.frame_size_ms = config.frame_size_ms;
impl_config.payload_type = payload_type;
return std::make_unique<AudioEncoderPcmU>(impl_config);
}
case Config::Type::kPcmA: {
AudioEncoderPcmA::Config impl_config;
impl_config.num_channels = config.num_channels;
impl_config.frame_size_ms = config.frame_size_ms;
impl_config.payload_type = payload_type;
return std::make_unique<AudioEncoderPcmA>(impl_config);
}
default: {
return nullptr;
}
}
}
我们以AudioEncoderG711的代码为例,分析具体逻辑。其中SdpToConfig 是 把sdp里面的audio format转换成AudioEncoderFactory可识别的config,从中检索出对应的AudioEncoder,这里涉及 perconnection里面的sdp交互,本文章是结构分析,往后结构分析结束之后再逻辑分析就能一目了然。
![](/qrcode.jpg)
如果你想自己添加新的音频编码支持(AAC),可以在AudioEncoderFactory,按照Helper的模板实现新的AudioEncoder。
AudioDecoderFactoryFactory 代码逻辑同理,就不在这里展开分析了。
二、AudioProcessingFactory
AudioProcessingFactory跟踪代码只是一个声明接口,在demo的代码中没有默认实现类,这只能顺着createPeerConnectionFactory进去看看有没有默认实现。
//org.webrtc.PeerConnectionFactory.java Builder
public PeerConnectionFactory createPeerConnectionFactory() {
PeerConnectionFactory.checkInitializeHasBeenCalled();
if (this.audioDeviceModule == null) {
this.audioDeviceModule = JavaAudioDeviceModule.builder(ContextUtils.getApplicationContext()).createAudioDeviceModule();
}
return PeerConnectionFactory.nativeCreatePeerConnectionFactory(
ContextUtils.getApplicationContext(),
this.options,
this.audioDeviceModule.getNativeAudioDeviceModulePointer(),
this.audioEncoderFactoryFactory.createNativeAudioEncoderFactory(),
this.audioDecoderFactoryFactory.createNativeAudioDecoderFactory(),
this.videoEncoderFactory,
this.videoDecoderFactory,
this.audioProcessingFactory == null ? 0L : this.audioProcessingFactory.createNative(),
this.fecControllerFactoryFactory == null ? 0L : this.fecControllerFactoryFactory.createNative(),
this.networkControllerFactoryFactory == null ? 0L : this.networkControllerFactoryFactory.createNativeNetworkControllerFactory(),
this.networkStatePredictorFactoryFactory == null ? 0L : this.networkStatePredictorFactoryFactory.createNativeNetworkStatePredictorFactory(),
this.neteqFactoryFactory == null ? 0L : this.neteqFactoryFactory.createNativeNetEqFactory()
);
}
// gen\sdk\android\generated_peerconnection_jni.h
JNI_GENERATOR_EXPORT jobject
Java_org_webrtc_PeerConnectionFactory_nativeCreatePeerConnectionFactory(
JNIEnv* env,
jclass jcaller,
jobject context,
jobject options,
jlong nativeAudioDeviceModule,
jlong audioEncoderFactory,
jlong audioDecoderFactory,
jobject encoderFactory,
jobject decoderFactory,
jlong nativeAudioProcessor,
jlong nativeFecControllerFactory,
jlong nativeNetworkControllerFactory,
jlong nativeNetworkStatePredictorFactory,
jlong neteqFactory) {
return JNI_PeerConnectionFactory_CreatePeerConnectionFactory(env,
base::android::JavaParamRef<jobject>(env, context), base::android::JavaParamRef<jobject>(env,options),
nativeAudioDeviceModule, audioEncoderFactory, audioDecoderFactory,
base::android::JavaParamRef<jobject>(env, encoderFactory),
base::android::JavaParamRef<jobject>(env, decoderFactory), nativeAudioProcessor,
nativeFecControllerFactory, nativeNetworkControllerFactory,
nativeNetworkStatePredictorFactory, neteqFactory).Release();
}
// sdk\android\src\jni\peer_connection_factory.cc
static ScopedJavaLocalRef<jobject>
JNI_PeerConnectionFactory_CreatePeerConnectionFactory(
JNIEnv* jni,
const JavaParamRef<jobject>& jcontext,
const JavaParamRef<jobject>& joptions,
jlong native_audio_device_module,
jlong native_audio_encoder_factory,
jlong native_audio_decoder_factory,
const JavaParamRef<jobject>& jencoder_factory,
const JavaParamRef<jobject>& jdecoder_factory,
jlong native_audio_processor,
jlong native_fec_controller_factory,
jlong native_network_controller_factory,
jlong native_network_state_predictor_factory,
jlong native_neteq_factory) {
rtc::scoped_refptr<AudioProcessing> audio_processor =
reinterpret_cast<AudioProcessing*>(native_audio_processor);
return CreatePeerConnectionFactoryForJava(
jni, jcontext, joptions,
reinterpret_cast<AudioDeviceModule*>(native_audio_device_module),
TakeOwnershipOfRefPtr<AudioEncoderFactory>(native_audio_encoder_factory),
TakeOwnershipOfRefPtr<AudioDecoderFactory>(native_audio_decoder_factory),
jencoder_factory, jdecoder_factory,
audio_processor ? audio_processor : CreateAudioProcessing(),
TakeOwnershipOfUniquePtr<FecControllerFactoryInterface>(native_fec_controller_factory),
TakeOwnershipOfUniquePtr<NetworkControllerFactoryInterface>(native_network_controller_factory),
TakeOwnershipOfUniquePtr<NetworkStatePredictorFactoryInterface>(native_network_state_predictor_factory),
TakeOwnershipOfUniquePtr<NetEqFactory>(native_neteq_factory));
}
跟踪代码发现,在CreatePeerConnectionFactoryForJava入参当中,audio_processor有一个判空的三元操作。 默认实现类找到了!
// sdk\android\src\jni\pc\audio.cc
rtc::scoped_refptr<AudioProcessing> CreateAudioProcessing() {
return AudioProcessingBuilder().Create();
}
// modules\audio_processing\audio_processing_builder_impl.cc
AudioProcessing* AudioProcessingBuilder::Create() {
webrtc::Config config;
return Create(config);
}
AudioProcessing* AudioProcessingBuilder::Create(const webrtc::Config& config) {
#ifdef WEBRTC_EXCLUDE_AUDIO_PROCESSING_MODULE
// Implementation returning a null pointer for using when the APM is excluded
// from the build..
return nullptr;
#else
// Standard implementation.
return new rtc::RefCountedObject<AudioProcessingImpl>(
config, std::move(capture_post_processing_),
std::move(render_pre_processing_), std::move(echo_control_factory_),
std::move(echo_detector_), std::move(capture_analyzer_));
#endif
}
快速跟踪代码就可以定位到默认实现AudioProcessingImpl,但是代码量实在太多不好拆分理解。我们不妨来看其父类AudioProcessing的注释介绍,以及测试代码。
// 音频处理模块(APM)提供一系列为实时通信软件设计的语音处理组件。
//
// APM在逐帧的基础上对两个音频流进行操作。
// 在应用上,逐一处理帧的主流是通过|ProcessStream()|传递。
// 反向的帧流是通过到| ProcessReverseStream()|传递。
// 在客户端,这通常是指近端(捕获)和远端(渲染)流。
// APM应放置在信号链中,尽可能靠近音频硬件抽象层(HAL)。
//
// 在服务器端,通常不使用反向流,在每个传入流上进行处理。
//
// 组件接口遵循类似的模式,可以通过APM中对应的getter方法。
// 所有组件都在创建时被禁用,默认设置建议用于大多数情况。
// 可以在不启用组件的情况下应用新设置。
// 启用组件会触发内存分配和初始化,以允许它开始处理流。
//
// 线程安全性提供了以下假设,以减少锁定开销:
// 1. 流的getter和setter调用 与 ProcessStream() 属於同一线程。
// 更准确地说,流函数永远不会与ProcessStream() 并发调用。
//
// 2. 参数getter永远不会与相应的setter同时调用。
//
// APM只接受10毫秒范围内的 一个chunk的线性PCM音频数据。
// int16入参类型的接口使用双通道交织数据,而浮点接口使用非交织数据。
//
// 以下是用法示例,省略错误检查:
// AudioProcessing* apm = AudioProcessingBuilder().Create();
//
// AudioProcessing::Config config;
// config.echo_canceller.enabled = true;
// config.echo_canceller.mobile_mode = false;
//
// config.gain_controller1.enabled = true;
// config.gain_controller1.mode =
// AudioProcessing::Config::GainController1::kAdaptiveAnalog;
// config.gain_controller1.analog_level_minimum = 0;
// config.gain_controller1.analog_level_maximum = 255;
//
// config.gain_controller2.enabled = true;
//
// config.high_pass_filter.enabled = true;
//
// config.voice_detection.enabled = true;
//
// apm->ApplyConfig(config)
//
// apm->noise_reduction()->set_level(kHighSuppression);
// apm->noise_reduction()->Enable(true);
//
// /*开始语音通话。。。*/
//
// // ... 待播放/渲染的帧传递到绑定的音频HAL ...
// apm->ProcessReverseStream(render_frame);
//
// // 调用所需的set_stream_函数。
// apm->set_stream_delay_ms(delay_ms);
// apm->set_stream_analog_level(analog_level);
//
// // ... 从音频HAL捕获帧 ...
// apm->ProcessStream(capture_frame);
//
// // 调用所需的set_stream_函数。
// analog_level = apm->recommended_stream_analog_level();
// has_voice = apm->stream_has_voice();
//
// // 在调用期间重复渲染和捕获处理。。。
// apm->Initialize();
//
// // 关闭应用程序。。。
// delete apm;
class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface {
struct RTC_EXPORT Config { ...
virtual void ApplyConfig(const Config& config) = 0;
virtual int proc_sample_rate_hz() const = 0;
virtual int proc_split_sample_rate_hz() const = 0;
virtual size_t num_input_channels() const = 0;
virtual size_t num_proc_channels() const = 0;
virtual size_t num_output_channels() const = 0;
virtual size_t num_reverse_channels() const = 0;
virtual int ProcessStream(const int16_t* const src,
const StreamConfig& input_config,
const StreamConfig& output_config,
int16_t* const dest) = 0;
virtual bool GetLinearAecOutput(
rtc::ArrayView<std::array<float, 160>> linear_output) const = 0;
virtual bool CreateAndAttachAecDump(const std::string& file_name,
int64_t max_log_size_bytes,
rtc::TaskQueue* worker_queue) = 0;
virtual void AttachAecDump(std::unique_ptr<AecDump> aec_dump) = 0;
virtual void DetachAecDump() = 0;
... ...
};
总结
音频模块结构分析暂且到这,加上上一章节的AudioDeviceModule(adm),和本章节的AudioEncoderFactory->AudioEncoder,AudioDecoderFactory->AudioDecoder,AudioProcessingFacfoty->AudioProcessing。音频共4个核心模块,负责的侧重点都各不相同。真对不同应用场景,需要进行不同方向的研究。