Skip to content

Commit dc87254

Browse files
authored
feat: universal resampler (#956)
1 parent 2b7b5c0 commit dc87254

45 files changed

Lines changed: 14869 additions & 500 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.gitignore

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,5 +97,4 @@ packages/react-native-audio-api/android/src/main/jniLibs/
9797
packages/react-native-audio-api/common/cpp/audioapi/external/**/*.a
9898
packages/react-native-audio-api/common/cpp/audioapi/external/*.xcframework
9999
packages/react-native-audio-api/common/cpp/audioapi/external/ffmpeg_ios/
100-
101100
.cache

packages/react-native-audio-api/RNAudioAPI.podspec

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ Pod::Spec.new do |s|
4141
end
4242

4343
ss.subspec "audioapi_dsp" do |sss|
44-
sss.source_files = "common/cpp/audioapi/dsp/**/*.{cpp}"
44+
sss.source_files = "common/cpp/audioapi/dsp/**/*.{cpp,inc}"
4545
sss.header_dir = "audioapi/dsp"
4646
sss.header_mappings_dir = "common/cpp/audioapi/dsp"
4747
sss.compiler_flags = "-O3"

packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioRecorder.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ Result<std::string, std::string> AndroidAudioRecorder::start(const std::string &
145145
if (isConnected()) {
146146
deinterleavingBuffer_ = std::make_shared<AudioBuffer>(
147147
streamMaxBufferSizeInFrames_, streamChannelCount_, streamSampleRate_);
148-
adapterNode_->init(streamMaxBufferSizeInFrames_, streamChannelCount_);
148+
adapterNode_->init(streamMaxBufferSizeInFrames_, streamChannelCount_, streamSampleRate_);
149149
}
150150

151151
auto result = mStream_->requestStart();
@@ -327,7 +327,7 @@ void AndroidAudioRecorder::connect(const std::shared_ptr<RecorderAdapterNode> &n
327327
if (!isIdle()) {
328328
deinterleavingBuffer_ = std::make_shared<AudioBuffer>(
329329
streamMaxBufferSizeInFrames_, streamChannelCount_, streamSampleRate_);
330-
adapterNode_->init(streamMaxBufferSizeInFrames_, streamChannelCount_);
330+
adapterNode_->init(streamMaxBufferSizeInFrames_, streamChannelCount_, streamSampleRate_);
331331
}
332332

333333
isConnected_.store(true, std::memory_order_release);

packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/utils/ffmpegBackend/FFmpegFileWriter.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ class FFmpegAudioFileWriter : public AndroidFileWriterBackend {
5555
Result<NoneType, std::string> configureAndOpenCodec(const AVCodec *codec);
5656
Result<NoneType, std::string> initializeStream();
5757
Result<NoneType, std::string> openIOAndWriteHeader();
58+
// TODO: rewrite to use r8brain resampler
5859
Result<NoneType, std::string> initializeResampler(float inputRate, int inputChannels);
5960
void initializeBuffers(int32_t maxBufferSize);
6061

packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/utils/miniaudioBackend/MiniAudioFileWriter.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ class MiniAudioFileWriter : public AndroidFileWriterBackend {
3333

3434
ma_result initializeConverterIfNeeded();
3535
ma_result initializeEncoder(const std::string &fileNameOverride);
36+
// TODO: rewrite to use r8brain resampler
3637
ma_uint64 convertBuffer(void *data, int numFrames);
3738

3839
bool isConverterRequired();

packages/react-native-audio-api/common/cpp/audioapi/core/effects/WaveShaperNode.cpp

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,7 @@
44
#include <audioapi/types/NodeOptions.h>
55
#include <audioapi/utils/AudioArrayBuffer.hpp>
66
#include <audioapi/utils/AudioBuffer.h>
7-
8-
#include <algorithm>
97
#include <memory>
10-
#include <string>
118

129
namespace audioapi {
1310

@@ -18,7 +15,7 @@ WaveShaperNode::WaveShaperNode(
1815

1916
waveShapers_.reserve(6);
2017
for (size_t i = 0; i < channelCount_; i++) {
21-
waveShapers_.emplace_back(std::make_unique<WaveShaper>(nullptr));
18+
waveShapers_.emplace_back(std::make_unique<WaveShaper>(nullptr, context->getSampleRate()));
2219
}
2320
setCurve(options.curve);
2421
isInitialized_ = true;

packages/react-native-audio-api/common/cpp/audioapi/core/effects/WaveShaperNode.h

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,11 @@
22

33
#include <audioapi/core/AudioNode.h>
44
#include <audioapi/core/types/OverSampleType.h>
5-
#include <audioapi/dsp/Resampler.h>
65
#include <audioapi/dsp/WaveShaper.h>
76

8-
#include <algorithm>
97
#include <atomic>
108
#include <memory>
119
#include <mutex>
12-
#include <string>
1310
#include <vector>
1411

1512
namespace audioapi {

packages/react-native-audio-api/common/cpp/audioapi/core/sources/RecorderAdapterNode.cpp

Lines changed: 83 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,12 @@
44
#include <audioapi/core/utils/Constants.h>
55
#include <audioapi/utils/AudioArray.h>
66
#include <audioapi/utils/AudioBuffer.h>
7+
8+
#include <algorithm>
9+
#include <cmath>
10+
#include <cstring>
711
#include <memory>
12+
#include <vector>
813

914
namespace audioapi {
1015

@@ -15,7 +20,7 @@ RecorderAdapterNode::RecorderAdapterNode(const std::shared_ptr<BaseAudioContext>
1520
isInitialized_ = false;
1621
}
1722

18-
void RecorderAdapterNode::init(size_t bufferSize, int channelCount) {
23+
void RecorderAdapterNode::init(size_t bufferSize, int channelCount, float sampleRate) {
1924
std::shared_ptr<BaseAudioContext> context = context_.lock();
2025
if (isInitialized_ || context == nullptr) {
2126
return;
@@ -29,26 +34,37 @@ void RecorderAdapterNode::init(size_t bufferSize, int channelCount) {
2934
buff_[i] = std::make_shared<CircularOverflowableAudioArray>(bufferSize);
3035
}
3136

32-
// This assumes that the sample rate is the same in audio context and recorder.
33-
// (recorder is not enforcing any sample rate on the system*). This means that only
34-
// channel mixing might be required. To do so, we create an output buffer with
35-
// the desired channel count and will take advantage of the AudioBuffer sum method.
36-
//
37-
// * any allocations required by the recorder (including this method) are during recording start
38-
// or after, which means that audio context has already setup the system in 99% of sane cases.
39-
// But if we would like to support cases when context is created on the fly during recording,
40-
// we would need to add sample rate conversion as well or other weird bullshit like resampling
41-
// context output and not enforcing anything on the system output/input configuration.
42-
// A lot of words for a couple of lines of implementation :shrug:
37+
float contextSampleRate = context->getSampleRate();
38+
needsResampling_ = static_cast<int>(sampleRate) != static_cast<int>(contextSampleRate);
39+
4340
adapterOutputBuffer_ =
44-
std::make_shared<AudioBuffer>(RENDER_QUANTUM_SIZE, channelCount_, context->getSampleRate());
41+
std::make_shared<AudioBuffer>(RENDER_QUANTUM_SIZE, channelCount_, contextSampleRate);
42+
43+
if (needsResampling_) {
44+
inputChunkSize_ =
45+
static_cast<size_t>(std::ceil(RENDER_QUANTUM_SIZE * sampleRate / contextSampleRate)) + 4;
46+
47+
resampler_ = std::make_unique<r8b::MultiChannelResampler>(
48+
sampleRate, contextSampleRate, channelCount_, static_cast<int>(inputChunkSize_));
49+
50+
const int maxOutLen = resampler_->getMaxOutLen();
51+
52+
resamplerInputBuffer_ = AudioBuffer(inputChunkSize_, channelCount_, sampleRate);
53+
resamplerOutputBuffer_ =
54+
AudioBuffer(static_cast<size_t>(maxOutLen), channelCount_, contextSampleRate);
55+
overflowBuffer_ = AudioBuffer(2 * maxOutLen, channelCount_, contextSampleRate);
56+
overflowSize_ = 0;
57+
}
58+
4559
isInitialized_ = true;
4660
}
4761

4862
void RecorderAdapterNode::cleanup() {
4963
isInitialized_ = false;
64+
needsResampling_ = false;
5065
buff_.clear();
51-
adapterOutputBuffer_.reset();
66+
resampler_.reset();
67+
overflowSize_ = 0;
5268
}
5369

5470
std::shared_ptr<AudioBuffer> RecorderAdapterNode::processNode(
@@ -59,17 +75,67 @@ std::shared_ptr<AudioBuffer> RecorderAdapterNode::processNode(
5975
return processingBuffer;
6076
}
6177

62-
readFrames(framesToProcess);
78+
if (needsResampling_) {
79+
processResampled(framesToProcess);
80+
} else {
81+
readFrames(*adapterOutputBuffer_, framesToProcess);
82+
}
6383

6484
processingBuffer->sum(*adapterOutputBuffer_, ChannelInterpretation::SPEAKERS);
6585
return processingBuffer;
6686
}
6787

68-
void RecorderAdapterNode::readFrames(const size_t framesToRead) {
88+
void RecorderAdapterNode::processResampled(int framesToProcess) {
6989
adapterOutputBuffer_->zero();
7090

91+
size_t outputWritten = 0;
92+
const size_t needed = static_cast<size_t>(framesToProcess);
93+
94+
// Drain leftover resampled samples from the previous call
95+
if (overflowSize_ > 0) {
96+
const size_t toCopy = std::min(overflowSize_, needed);
97+
98+
adapterOutputBuffer_->copy(overflowBuffer_, 0, 0, toCopy);
99+
outputWritten = toCopy;
100+
101+
if (toCopy < overflowSize_) {
102+
const size_t remaining = overflowSize_ - toCopy;
103+
for (int ch = 0; ch < channelCount_; ++ch) {
104+
overflowBuffer_[ch].copyWithin(toCopy, 0, remaining);
105+
}
106+
}
107+
overflowSize_ -= toCopy;
108+
}
109+
110+
// Feed the resampler until we have enough output frames
111+
while (outputWritten < needed) {
112+
readFrames(resamplerInputBuffer_, inputChunkSize_);
113+
114+
const int outLen = resampler_->process(
115+
resamplerInputBuffer_, static_cast<int>(inputChunkSize_), resamplerOutputBuffer_);
116+
117+
const size_t remaining = needed - outputWritten;
118+
const size_t toCopy = std::min(static_cast<size_t>(outLen), remaining);
119+
120+
// Write resampled frames into the output buffer
121+
adapterOutputBuffer_->copy(resamplerOutputBuffer_, 0, outputWritten, toCopy);
122+
outputWritten += toCopy;
123+
124+
// Stash excess for the next processNode call
125+
const int excess = outLen - static_cast<int>(toCopy);
126+
if (excess > 0) {
127+
overflowBuffer_.copy(
128+
resamplerOutputBuffer_, toCopy, overflowSize_, static_cast<size_t>(excess));
129+
overflowSize_ += static_cast<size_t>(excess);
130+
}
131+
}
132+
}
133+
134+
void RecorderAdapterNode::readFrames(AudioBuffer &target, const size_t framesToRead) {
135+
target.zero();
136+
71137
for (size_t channel = 0; channel < channelCount_; ++channel) {
72-
buff_[channel]->read(*adapterOutputBuffer_->getChannel(channel), framesToRead);
138+
buff_[channel]->read(*target.getChannel(channel), framesToRead);
73139
}
74140
}
75141

packages/react-native-audio-api/common/cpp/audioapi/core/sources/RecorderAdapterNode.h

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <audioapi/core/AudioParam.h>
55
#include <audioapi/core/BaseAudioContext.h>
66
#include <audioapi/core/inputs/AudioRecorder.h>
7+
#include <audioapi/dsp/r8brain/Resampler.h>
78
#include <audioapi/utils/CircularOverflowableAudioArray.h>
89
#include <memory>
910
#include <vector>
@@ -25,7 +26,8 @@ class RecorderAdapterNode : public AudioNode {
2526
/// @note This method should be called ONLY ONCE when the buffer size is known.
2627
/// @param bufferSize The size of the buffer to be used.
2728
/// @param channelCount The number of channels.
28-
void init(size_t bufferSize, int channelCount);
29+
/// @param sampleRate The recorder's native sample rate.
30+
void init(size_t bufferSize, int channelCount, float sampleRate);
2931
void cleanup();
3032

3133
int channelCount_{};
@@ -39,10 +41,20 @@ class RecorderAdapterNode : public AudioNode {
3941
std::shared_ptr<AudioBuffer> adapterOutputBuffer_;
4042

4143
private:
42-
/// @brief Read audio frames from the recorder's internal circular buffer into output buffers.
43-
/// @note If `framesToRead` is greater than the number of available frames, it will fill empty space with silence.
44-
/// @param framesToRead Number of frames to read.
45-
void readFrames(size_t framesToRead);
44+
void readFrames(AudioBuffer &target, size_t framesToRead);
45+
void processResampled(int framesToProcess);
46+
47+
std::unique_ptr<r8b::MultiChannelResampler> resampler_;
48+
bool needsResampling_ = false;
49+
50+
// Number of input frames (at recorder rate) to feed per render quantum
51+
size_t inputChunkSize_ = 0;
52+
AudioBuffer resamplerInputBuffer_;
53+
AudioBuffer resamplerOutputBuffer_;
54+
55+
// Accumulates resampled output across calls
56+
AudioBuffer overflowBuffer_;
57+
size_t overflowSize_ = 0;
4658
};
4759

4860
} // namespace audioapi

0 commit comments

Comments
 (0)