astra/prism
1
0
Fork 0
mirror of https://github.com/Boof2015/prism.git synced 2026-08-18 19:44:16 +02:00
Code Issues Packages Projects Releases Wiki Activity

make Spectrum and Spectrogram more frequency accurate and detail preserving

Browse Source
This commit is contained in:
Boof2015
2026-08-16 16:43:31 -04:00
parent f6d9719c43
commit 3b302d5526
29 changed files with 1715 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files
native/src/main.cpp
+34 -13
View File
@@ -327,6 +327,27 @@ std::string GetObjectString(const Napi::Object& obj, const char* key, const std:
Napi::Value value = obj.Get(key);
return value.IsString() ? value.As<Napi::String>().Utf8Value() : fallback;
}
Napi::Object SpectrogramResultToJs(
Napi::Env env,
const Visualizer::SpectrogramProcessResult& result
) {
Napi::Float32Array display = Napi::Float32Array::New(env, result.display.size());
Napi::Float32Array heat = Napi::Float32Array::New(env, result.heat.size());
if (!result.display.empty()) {
memcpy(display.Data(), result.display.data(), result.display.size() * sizeof(float));
}
if (!result.heat.empty()) {
memcpy(heat.Data(), result.heat.data(), result.heat.size() * sizeof(float));
}
Napi::Object obj = Napi::Object::New(env);
obj.Set("display", display);
obj.Set("heat", heat);
obj.Set("columnCount", Napi::Number::New(env, static_cast<double>(result.columnCount)));
obj.Set("rowCount", Napi::Number::New(env, static_cast<double>(result.rowCount)));
return obj;
}
} // namespace
Napi::Value SpectrogramConfigure(const Napi::CallbackInfo& info) {
@@ -365,22 +386,21 @@ Napi::Value SpectrogramProcess(const Napi::CallbackInfo& info) {
Napi::Float32Array audioData = info[0].As<Napi::Float32Array>();
auto result = spectrogramAnalyzer.process(audioData.Data(), audioData.ElementLength());
return SpectrogramResultToJs(env, result);
}
Napi::Float32Array display = Napi::Float32Array::New(env, result.display.size());
Napi::Float32Array heat = Napi::Float32Array::New(env, result.heat.size());
if (!result.display.empty()) {
memcpy(display.Data(), result.display.data(), result.display.size() * sizeof(float));
}
if (!result.heat.empty()) {
memcpy(heat.Data(), result.heat.data(), result.heat.size() * sizeof(float));
Napi::Value SpectrogramProcessStereo(const Napi::CallbackInfo& info) {
Napi::Env env = info.Env();
if (info.Length() < 2 || !info[0].IsTypedArray() || !info[1].IsTypedArray()) {
Napi::TypeError::New(env, "Expected left and right Float32Arrays").ThrowAsJavaScriptException();
return env.Null();
}
Napi::Object obj = Napi::Object::New(env);
obj.Set("display", display);
obj.Set("heat", heat);
obj.Set("columnCount", Napi::Number::New(env, static_cast<double>(result.columnCount)));
obj.Set("rowCount", Napi::Number::New(env, static_cast<double>(result.rowCount)));
return obj;
Napi::Float32Array left = info[0].As<Napi::Float32Array>();
Napi::Float32Array right = info[1].As<Napi::Float32Array>();
const size_t length = std::min(left.ElementLength(), right.ElementLength());
auto result = spectrogramAnalyzer.processStereo(left.Data(), right.Data(), length);
return SpectrogramResultToJs(env, result);
}
Napi::Value SpectrogramReset(const Napi::CallbackInfo& info) {
@@ -723,6 +743,7 @@ Napi::Object Init(Napi::Env env, Napi::Object exports) {
Napi::Object spectrogramExports = Napi::Object::New(env);
spectrogramExports.Set("configure", Napi::Function::New(env, SpectrogramConfigure));
spectrogramExports.Set("process", Napi::Function::New(env, SpectrogramProcess));
spectrogramExports.Set("processStereo", Napi::Function::New(env, SpectrogramProcessStereo));
spectrogramExports.Set("reset", Napi::Function::New(env, SpectrogramReset));
exports.Set("spectrogram", spectrogramExports);
native/src/spectrogram.cpp
+298 -86
View File
@@ -9,10 +9,10 @@ namespace Visualizer {
namespace {
constexpr size_t FFT_PAD_FACTOR = 4;
constexpr float DISPLAY_GAIN_DB = 2.0f;
constexpr float HANN_EQUIVALENT_NOISE_BANDWIDTH_BINS = 1.5f;
constexpr float REASSIGNED_POWER_NORMALIZATION = static_cast<float>(FFT_PAD_FACTOR)
* HANN_EQUIVALENT_NOISE_BANDWIDTH_BINS;
constexpr float SPECTROGRAM_HEAT_GAMMA = 1.45f;
constexpr float SPECTROGRAM_DISPLAY_STROKE_WEIGHT = 0.42f;
constexpr float SPECTROGRAM_HEAT_STROKE_WEIGHT = 0.32f;
constexpr float TILT_REFERENCE_HZ = 1000.0f;
constexpr float HEAT_MIN_DB = -100.0f;
constexpr float HEAT_MAX_DB = -20.0f;
@@ -64,7 +64,8 @@ SpectrogramAnalyzer::SpectrogramAnalyzer()
: fftSize_(0)
, paddedSize_(0)
, frameFill_(0)
, haveLastPhase_(false) {
, haveLastPhase_(false)
, magnitudeScale_(1.0f) {
configureFft(config_.fftSize);
rebuildFrequencyMapping();
}
@@ -96,7 +97,10 @@ void SpectrogramAnalyzer::configure(const SpectrogramConfig& config) {
if (next.orientation != "vertical") {
next.orientation = "horizontal";
}
if (next.clarityMode != "classic" && next.clarityMode != "sharp" && next.clarityMode != "sharper") {
if (next.clarityMode != "classic"
&& next.clarityMode != "focused"
&& next.clarityMode != "sharp"
&& next.clarityMode != "sharper") {
next.clarityMode = "sharper";
}
@@ -128,13 +132,19 @@ void SpectrogramAnalyzer::configureFft(size_t fftSize) {
paddedSize_ = fftSize_ * FFT_PAD_FACTOR;
fft_ = std::make_unique<DSP::FFT>(paddedSize_);
frameBuffer_.assign(fftSize_, 0.0f);
rightFrameBuffer_.assign(fftSize_, 0.0f);
window_.assign(fftSize_, 1.0f);
windowedInput_.assign(paddedSize_, 0.0f);
rightWindowedInput_.assign(paddedSize_, 0.0f);
fftOutput_.assign(paddedSize_, std::complex<float>(0.0f, 0.0f));
rightFftOutput_.assign(paddedSize_, std::complex<float>(0.0f, 0.0f));
magnitudesDb_.assign(paddedSize_ / 2, -200.0f);
magnitudesLinear_.assign(paddedSize_ / 2, 0.0f);
phases_.assign(paddedSize_ / 2, 0.0f);
lastPhases_.assign(paddedSize_ / 2, 0.0f);
rightPhases_.assign(paddedSize_ / 2, 0.0f);
rightLastPhases_.assign(paddedSize_ / 2, 0.0f);
dominantRight_.assign(paddedSize_ / 2, 0);
frameFill_ = 0;
haveLastPhase_ = false;
@@ -145,11 +155,21 @@ void SpectrogramAnalyzer::configureFft(size_t fftSize) {
for (size_t index = 0; index < fftSize_; index += 1) {
window_[index] = 0.5f * (1.0f - std::cos((2.0f * static_cast<float>(M_PI) * index) / (fftSize_ - 1)));
}
float windowSum = 0.0f;
for (const float coefficient : window_) {
windowSum += coefficient;
}
magnitudeScale_ = windowSum > std::numeric_limits<float>::epsilon()
? 2.0f / windowSum
: 1.0f;
}
void SpectrogramAnalyzer::reset() {
std::fill(frameBuffer_.begin(), frameBuffer_.end(), 0.0f);
std::fill(rightFrameBuffer_.begin(), rightFrameBuffer_.end(), 0.0f);
std::fill(lastPhases_.begin(), lastPhases_.end(), 0.0f);
std::fill(rightLastPhases_.begin(), rightLastPhases_.end(), 0.0f);
frameFill_ = 0;
haveLastPhase_ = false;
}
@@ -166,7 +186,7 @@ void SpectrogramAnalyzer::rebuildFrequencyMapping() {
const float sampleRate = std::max(1.0f, config_.sampleRate);
const float nyquist = sampleRate * 0.5f;
const float minFrequency = std::max(1.0f, std::min(config_.minFrequency, nyquist));
const float maxFrequency = std::max(minFrequency + 1.0f, std::min(config_.maxFrequency, nyquist));
const float maxFrequency = std::max(minFrequency, std::min(config_.maxFrequency, nyquist));
config_.minFrequency = minFrequency;
config_.maxFrequency = maxFrequency;
@@ -177,8 +197,9 @@ void SpectrogramAnalyzer::rebuildFrequencyMapping() {
standardRaw_.assign(rowCount, 0.0f);
standardHeat_.assign(rowCount, 0.0f);
reassignedPower_.assign(rowCount, 0.0f);
blendedRaw_.assign(rowCount, 0.0f);
blendedHeat_.assign(rowCount, 0.0f);
focusedPower_.assign(rowCount, 0.0f);
sourceRaw_.assign(rowCount, 0.0f);
sourceHeat_.assign(rowCount, 0.0f);
shapedDisplay_.assign(rowCount, 0.0f);
shapedHeat_.assign(rowCount, 0.0f);
strokedDisplay_.assign(rowCount, 0.0f);
@@ -218,7 +239,11 @@ void SpectrogramAnalyzer::rebuildFrequencyMapping() {
float SpectrogramAnalyzer::frequencyFromScale(float normalizedPosition) const {
const float t = clamp01(normalizedPosition);
const float minFrequency = std::max(1.0f, config_.minFrequency);
const float maxFrequency = std::max(minFrequency + 1.0f, config_.maxFrequency);
const float maxFrequency = std::max(minFrequency, config_.maxFrequency);
if (maxFrequency <= minFrequency) {
return minFrequency;
}
if (config_.scaleMode == "linear") {
return minFrequency + (t * (maxFrequency - minFrequency));
@@ -236,8 +261,15 @@ float SpectrogramAnalyzer::frequencyFromScale(float normalizedPosition) const {
}
float SpectrogramAnalyzer::frequencyToRow(float frequency) const {
if (config_.rowCount <= 1) {
return 0.0f;
}
const float minFrequency = std::max(1.0f, config_.minFrequency);
const float maxFrequency = std::max(minFrequency + 1.0f, config_.maxFrequency);
const float maxFrequency = std::max(minFrequency, config_.maxFrequency);
if (maxFrequency <= minFrequency) {
return config_.orientation == "vertical" ? 0.0f : static_cast<float>(config_.rowCount - 1);
}
const float clampedFrequency = std::clamp(frequency, minFrequency, maxFrequency);
float normalized = 0.0f;
@@ -262,7 +294,7 @@ float SpectrogramAnalyzer::frequencyToRow(float frequency) const {
float SpectrogramAnalyzer::applyDisplayTilt(float db, float frequency) const {
const float safeFrequency = std::max(1.0f, frequency);
const float tiltAmount = config_.tiltDbPerOctave * std::log2(safeFrequency / TILT_REFERENCE_HZ);
return db + tiltAmount + DISPLAY_GAIN_DB;
return db + tiltAmount;
}
float SpectrogramAnalyzer::displayDbToIntensity(float db) const {
@@ -296,10 +328,44 @@ float SpectrogramAnalyzer::sampleDbAtBin(float bin) const {
);
}
float SpectrogramAnalyzer::samplePeakDbInBand(float startBin, float endBin, float& peakBin) const {
if (magnitudesDb_.empty()) {
peakBin = 0.0f;
return -200.0f;
}
const float lastBin = static_cast<float>(magnitudesDb_.size() - 1);
const float lo = std::clamp(std::min(startBin, endBin), 0.0f, lastBin);
const float hi = std::clamp(std::max(startBin, endBin), 0.0f, lastBin);
peakBin = 0.5f * (lo + hi);
float peakDb = sampleDbAtBin(peakBin);
const auto consider = [&](float candidateBin) {
const float candidateDb = sampleDbAtBin(candidateBin);
if (candidateDb > peakDb) {
peakDb = candidateDb;
peakBin = candidateBin;
}
};
consider(lo);
consider(hi);
const size_t firstWholeBin = static_cast<size_t>(std::ceil(lo));
const size_t lastWholeBin = static_cast<size_t>(std::floor(hi));
for (size_t bin = firstWholeBin; bin <= lastWholeBin && bin < magnitudesDb_.size(); bin += 1) {
consider(static_cast<float>(bin));
}
return peakDb;
}
void SpectrogramAnalyzer::computeStandardSpectrum() {
const size_t rowCount = config_.rowCount;
const float binWidth = std::max(1.0f, config_.sampleRate) / static_cast<float>(paddedSize_);
for (size_t row = 0; row < rowCount; row += 1) {
const float displayDb = applyDisplayTilt(sampleDbAtBin(rowCenterBins_[row]), rowCenterFrequencies_[row]);
float peakBin = rowCenterBins_[row];
const float rawDb = samplePeakDbInBand(rowBandStartBins_[row], rowBandEndBins_[row], peakBin);
const float displayDb = applyDisplayTilt(rawDb, peakBin * binWidth);
standardRaw_[row] = displayDbToIntensity(displayDb);
standardHeat_[row] = normalizeHeatDb(displayDb);
}
@@ -314,7 +380,11 @@ void SpectrogramAnalyzer::computeReassignedSpectrum() {
const float sampleRate = std::max(1.0f, config_.sampleRate);
const float binWidth = sampleRate / static_cast<float>(paddedSize_);
const float hopDt = static_cast<float>(resolveHopSize()) / sampleRate;
const float ampThreshold = std::pow(10.0f, config_.minDecibels / 20.0f);
// Reassign every bin that can contribute to either output. Limiting this to
// local maxima throws away low-level partials and ambience — exactly the
// detail a sharpened spectrogram is meant to retain.
const float visibleFloorDb = std::min(config_.minDecibels, HEAT_MIN_DB);
const float ampThreshold = std::pow(10.0f, visibleFloorDb / 20.0f);
const float twoPi = static_cast<float>(2.0 * M_PI);
for (size_t bin = 1; bin + 1 < magnitudesLinear_.size(); bin += 1) {
@@ -322,9 +392,6 @@ void SpectrogramAnalyzer::computeReassignedSpectrum() {
if (mag <= ampThreshold) {
continue;
}
if (mag < magnitudesLinear_[bin - 1] || mag < magnitudesLinear_[bin + 1]) {
continue;
}
const float nominalFrequency = static_cast<float>(bin) * binWidth;
if (nominalFrequency < config_.minFrequency || nominalFrequency > config_.maxFrequency) {
@@ -332,9 +399,67 @@ void SpectrogramAnalyzer::computeReassignedSpectrum() {
}
const float expected = twoPi * nominalFrequency * hopDt;
float correctionHz = wrapPhase(phases_[bin] - lastPhases_[bin] - expected) / (twoPi * hopDt);
const bool useRightPhase = dominantRight_[bin] != 0;
const float currentPhase = useRightPhase ? rightPhases_[bin] : phases_[bin];
const float previousPhase = useRightPhase ? rightLastPhases_[bin] : lastPhases_[bin];
const float correctionHz = wrapPhase(currentPhase - previousPhase - expected) / (twoPi * hopDt);
const float reassignedFrequency = nominalFrequency + correctionHz;
if (reassignedFrequency < config_.minFrequency || reassignedFrequency > config_.maxFrequency) {
continue;
}
const float rowF = frequencyToRow(reassignedFrequency);
const size_t row0 = static_cast<size_t>(std::floor(std::clamp(rowF, 0.0f, static_cast<float>(config_.rowCount - 1))));
const float frac = rowF - static_cast<float>(row0);
// A coherently-normalized Hann spectrum contains 1.5 bins of equivalent
// noise bandwidth. With 4x zero padding, a bin-centered sinusoid therefore
// contributes 6x its signal power across the positive-frequency bins.
// Divide that back out so relocation conserves calibrated signal power.
const float power = (mag * mag) / REASSIGNED_POWER_NORMALIZATION;
reassignedPower_[row0] += power * (1.0f - frac);
if (row0 + 1 < config_.rowCount) {
reassignedPower_[row0 + 1] += power * frac;
}
}
}
void SpectrogramAnalyzer::computeFocusedSpectrum() {
std::fill(focusedPower_.begin(), focusedPower_.end(), 0.0f);
if (!haveLastPhase_ || magnitudesLinear_.size() < 3 || config_.rowCount == 0) {
return;
}
const float sampleRate = std::max(1.0f, config_.sampleRate);
const float binWidth = sampleRate / static_cast<float>(paddedSize_);
const float hopDt = static_cast<float>(resolveHopSize()) / sampleRate;
const float ampThreshold = std::pow(10.0f, config_.minDecibels / 20.0f);
const float twoPi = static_cast<float>(2.0 * M_PI);
// Focused intentionally restores Prism's former peak-isolation aesthetic:
// only local FFT maxima are relocated, with conservative phase correction
// and a local spectral centroid. It is kept separate from Sharp/Sharper so
// their energy-preserving reassignment cannot silently lose texture.
for (size_t bin = 1; bin + 1 < magnitudesLinear_.size(); bin += 1) {
const float mag = magnitudesLinear_[bin];
if (mag <= ampThreshold
|| mag < magnitudesLinear_[bin - 1]
|| mag < magnitudesLinear_[bin + 1]) {
continue;
}
const float nominalFrequency = static_cast<float>(bin) * binWidth;
if (nominalFrequency < config_.minFrequency || nominalFrequency > config_.maxFrequency) {
continue;
}
const float expected = twoPi * nominalFrequency * hopDt;
const bool useRightPhase = dominantRight_[bin] != 0;
const float currentPhase = useRightPhase ? rightPhases_[bin] : phases_[bin];
const float previousPhase = useRightPhase ? rightLastPhases_[bin] : lastPhases_[bin];
float correctionHz = wrapPhase(currentPhase - previousPhase - expected) / (twoPi * hopDt);
correctionHz = std::clamp(correctionHz, -1.5f * binWidth, 1.5f * binWidth);
float reassignedFrequency = nominalFrequency + correctionHz;
float focusedFrequency = nominalFrequency + correctionHz;
const float leftWeight = magnitudesLinear_[bin - 1];
const float centerWeight = mag;
@@ -346,97 +471,163 @@ void SpectrogramAnalyzer::computeReassignedSpectrum() {
+ (nominalFrequency * centerWeight)
+ (static_cast<float>(bin + 1) * binWidth * rightWeight)
) / weightSum;
reassignedFrequency = 0.5f * reassignedFrequency + 0.5f * centroidFrequency;
focusedFrequency = 0.5f * focusedFrequency + 0.5f * centroidFrequency;
}
reassignedFrequency = std::clamp(reassignedFrequency, config_.minFrequency, config_.maxFrequency);
const float rowF = frequencyToRow(reassignedFrequency);
const size_t row0 = static_cast<size_t>(std::floor(std::clamp(rowF, 0.0f, static_cast<float>(config_.rowCount - 1))));
if (focusedFrequency < config_.minFrequency || focusedFrequency > config_.maxFrequency) {
continue;
}
const float rowF = frequencyToRow(focusedFrequency);
const size_t row0 = static_cast<size_t>(std::floor(
std::clamp(rowF, 0.0f, static_cast<float>(config_.rowCount - 1))
));
const float frac = rowF - static_cast<float>(row0);
const float power = mag * mag;
reassignedPower_[row0] += power * (1.0f - frac);
focusedPower_[row0] += power * (1.0f - frac);
if (row0 + 1 < config_.rowCount) {
reassignedPower_[row0 + 1] += power * frac;
focusedPower_[row0 + 1] += power * frac;
}
}
}
SpectrogramAnalyzer::ClarityProfile SpectrogramAnalyzer::clarityProfile(const std::string& mode) {
if (mode == "classic") {
return {1.4f, 0.0f, 3.0f};
return {1.4f, 0.42f, 0.32f, false};
}
if (mode == "sharp") {
return {1.5f, 2.5f, 3.0f};
return {1.25f, 0.22f, 0.16f, true};
}
return {2.0f, 5.0f, 2.0f};
return {1.1f, 0.08f, 0.06f, true};
}
void SpectrogramAnalyzer::blendAndShapeColumn(std::vector<float>& display, std::vector<float>& heat) {
void SpectrogramAnalyzer::shapeColumn(std::vector<float>& display, std::vector<float>& heat) {
if (config_.clarityMode == "focused") {
shapeFocusedColumn(display, heat);
return;
}
const size_t rowCount = config_.rowCount;
const ClarityProfile clarity = clarityProfile(config_.clarityMode);
const float standardWeight = config_.clarityMode == "classic" ? 0.8f : (config_.clarityMode == "sharp" ? 0.6f : 0.45f);
const float reassignedWeight = config_.clarityMode == "classic" ? 0.85f : 1.0f;
const bool useReassignedColumn = clarity.useReassignment && haveLastPhase_;
for (size_t row = 0; row < rowCount; row += 1) {
float reassignedRaw = 0.0f;
float reassignedHeat = 0.0f;
if (reassignedPower_[row] > 0.0f) {
if (useReassignedColumn && reassignedPower_[row] > 0.0f) {
const float reassignedMag = std::sqrt(reassignedPower_[row]);
const float reassignedDb = 20.0f * std::log10(std::max(reassignedMag, 1.0e-10f));
const float displayDb = applyDisplayTilt(reassignedDb, rowCenterFrequencies_[row]);
reassignedRaw = displayDbToIntensity(displayDb);
reassignedHeat = normalizeHeatDb(displayDb);
}
blendedRaw_[row] = std::max(standardRaw_[row] * standardWeight, reassignedRaw * reassignedWeight);
blendedHeat_[row] = std::max(standardHeat_[row] * standardWeight, reassignedHeat * reassignedWeight);
}
if (clarity.sharpness > 0.0f) {
const std::vector<float> peakSource = blendedRaw_;
const float mainlobePaddedBins = 4.0f * static_cast<float>(FFT_PAD_FACTOR);
const float detailPreserve = config_.clarityMode == "sharp" ? 0.18f : 0.14f;
for (size_t row = 0; row < rowCount; row += 1) {
const float bandWidthPerRow = std::max(0.1f, rowBandEndBins_[row] - rowBandStartBins_[row]);
const float mainlobePixels = mainlobePaddedBins / bandWidthPerRow;
const int halfWindow = std::max(2, std::min(50, static_cast<int>(std::lround(mainlobePixels * 0.5f))));
const float scaleFactor = std::max(1.0f, mainlobePixels / clarity.lineWidth);
const float effectiveSharpness = clarity.sharpness * scaleFactor;
float localMax = peakSource[row];
for (int offset = 1; offset <= halfWindow; offset += 1) {
if (row >= static_cast<size_t>(offset)) {
localMax = std::max(localMax, peakSource[row - static_cast<size_t>(offset)]);
}
if (row + static_cast<size_t>(offset) < rowCount) {
localMax = std::max(localMax, peakSource[row + static_cast<size_t>(offset)]);
}
}
if (localMax > 1.0e-6f) {
const float ratio = blendedRaw_[row] / localMax;
const float suppression = std::pow(clamp01(ratio), effectiveSharpness);
const float rawBefore = blendedRaw_[row];
const float heatBefore = blendedHeat_[row];
blendedRaw_[row] = std::max(rawBefore * suppression, rawBefore * detailPreserve);
blendedHeat_[row] = std::max(heatBefore * suppression, heatBefore * detailPreserve);
}
sourceRaw_[row] = displayDbToIntensity(displayDb);
sourceHeat_[row] = normalizeHeatDb(displayDb);
} else if (!useReassignedColumn) {
// The first phase-history frame falls back to Classic. Subsequent Sharp
// and Sharper frames contain only reassigned energy — no hidden Classic
// layer and no local-contrast gate.
sourceRaw_[row] = standardRaw_[row];
sourceHeat_[row] = standardHeat_[row];
} else {
sourceRaw_[row] = 0.0f;
sourceHeat_[row] = 0.0f;
}
}
const float effectiveGamma = clarity.gamma * config_.contrast;
for (size_t row = 0; row < rowCount; row += 1) {
shapedDisplay_[row] = std::pow(clamp01(blendedRaw_[row]), effectiveGamma);
shapedHeat_[row] = std::pow(clamp01(blendedHeat_[row]), SPECTROGRAM_HEAT_GAMMA);
shapedDisplay_[row] = std::pow(clamp01(sourceRaw_[row]), effectiveGamma);
shapedHeat_[row] = std::pow(clamp01(sourceHeat_[row]), SPECTROGRAM_HEAT_GAMMA);
strokedDisplay_[row] = shapedDisplay_[row];
strokedHeat_[row] = shapedHeat_[row];
}
for (size_t row = 0; row < rowCount; row += 1) {
const float displayShoulder = shapedDisplay_[row] * SPECTROGRAM_DISPLAY_STROKE_WEIGHT;
const float heatShoulder = shapedHeat_[row] * SPECTROGRAM_HEAT_STROKE_WEIGHT;
const float displayShoulder = shapedDisplay_[row] * clarity.displayShoulder;
const float heatShoulder = shapedHeat_[row] * clarity.heatShoulder;
if (row > 0) {
strokedDisplay_[row - 1] = std::max(strokedDisplay_[row - 1], displayShoulder);
strokedHeat_[row - 1] = std::max(strokedHeat_[row - 1], heatShoulder);
}
if (row + 1 < rowCount) {
strokedDisplay_[row + 1] = std::max(strokedDisplay_[row + 1], displayShoulder);
strokedHeat_[row + 1] = std::max(strokedHeat_[row + 1], heatShoulder);
}
}
const size_t offset = display.size();
display.resize(offset + rowCount);
heat.resize(offset + rowCount);
for (size_t row = 0; row < rowCount; row += 1) {
display[offset + row] = strokedDisplay_[row];
heat[offset + row] = strokedHeat_[row];
}
}
void SpectrogramAnalyzer::shapeFocusedColumn(std::vector<float>& display, std::vector<float>& heat) {
const size_t rowCount = config_.rowCount;
constexpr float standardWeight = 0.45f;
constexpr float reassignedWeight = 1.0f;
constexpr float sharpness = 5.0f;
constexpr float lineWidth = 2.0f;
constexpr float detailPreserve = 0.14f;
constexpr float gamma = 2.0f;
constexpr float displayShoulderWeight = 0.42f;
constexpr float heatShoulderWeight = 0.32f;
for (size_t row = 0; row < rowCount; row += 1) {
float focusedRaw = 0.0f;
float focusedHeat = 0.0f;
if (focusedPower_[row] > 0.0f) {
const float focusedMag = std::sqrt(focusedPower_[row]);
const float focusedDb = 20.0f * std::log10(std::max(focusedMag, 1.0e-10f));
const float displayDb = applyDisplayTilt(focusedDb, rowCenterFrequencies_[row]);
focusedRaw = displayDbToIntensity(displayDb);
focusedHeat = normalizeHeatDb(displayDb);
}
sourceRaw_[row] = std::max(standardRaw_[row] * standardWeight, focusedRaw * reassignedWeight);
sourceHeat_[row] = std::max(standardHeat_[row] * standardWeight, focusedHeat * reassignedWeight);
}
const std::vector<float> peakSource = sourceRaw_;
const float mainlobePaddedBins = 4.0f * static_cast<float>(FFT_PAD_FACTOR);
for (size_t row = 0; row < rowCount; row += 1) {
const float bandWidthPerRow = std::max(0.1f, rowBandEndBins_[row] - rowBandStartBins_[row]);
const float mainlobePixels = mainlobePaddedBins / bandWidthPerRow;
const int halfWindow = std::max(
2,
std::min(50, static_cast<int>(std::lround(mainlobePixels * 0.5f)))
);
const float scaleFactor = std::max(1.0f, mainlobePixels / lineWidth);
const float effectiveSharpness = sharpness * scaleFactor;
float localMax = peakSource[row];
for (int offset = 1; offset <= halfWindow; offset += 1) {
if (row >= static_cast<size_t>(offset)) {
localMax = std::max(localMax, peakSource[row - static_cast<size_t>(offset)]);
}
if (row + static_cast<size_t>(offset) < rowCount) {
localMax = std::max(localMax, peakSource[row + static_cast<size_t>(offset)]);
}
}
if (localMax > 1.0e-6f) {
const float suppression = std::pow(clamp01(sourceRaw_[row] / localMax), effectiveSharpness);
const float rawBefore = sourceRaw_[row];
const float heatBefore = sourceHeat_[row];
sourceRaw_[row] = std::max(rawBefore * suppression, rawBefore * detailPreserve);
sourceHeat_[row] = std::max(heatBefore * suppression, heatBefore * detailPreserve);
}
}
const float effectiveGamma = gamma * config_.contrast;
for (size_t row = 0; row < rowCount; row += 1) {
shapedDisplay_[row] = std::pow(clamp01(sourceRaw_[row]), effectiveGamma);
shapedHeat_[row] = std::pow(clamp01(sourceHeat_[row]), SPECTROGRAM_HEAT_GAMMA);
strokedDisplay_[row] = shapedDisplay_[row];
strokedHeat_[row] = shapedHeat_[row];
}
for (size_t row = 0; row < rowCount; row += 1) {
const float displayShoulder = shapedDisplay_[row] * displayShoulderWeight;
const float heatShoulder = shapedHeat_[row] * heatShoulderWeight;
if (row > 0) {
strokedDisplay_[row - 1] = std::max(strokedDisplay_[row - 1], displayShoulder);
strokedHeat_[row - 1] = std::max(strokedHeat_[row - 1], heatShoulder);
@@ -458,35 +649,54 @@ void SpectrogramAnalyzer::blendAndShapeColumn(std::vector<float>& display, std::
void SpectrogramAnalyzer::processFrame(std::vector<float>& display, std::vector<float>& heat) {
std::fill(windowedInput_.begin(), windowedInput_.end(), 0.0f);
std::fill(rightWindowedInput_.begin(), rightWindowedInput_.end(), 0.0f);
for (size_t index = 0; index < fftSize_; index += 1) {
windowedInput_[index] = frameBuffer_[index] * window_[index];
rightWindowedInput_[index] = rightFrameBuffer_[index] * window_[index];
}
fft_->forward(windowedInput_.data(), fftOutput_.data());
fft_->forward(rightWindowedInput_.data(), rightFftOutput_.data());
const size_t numBins = paddedSize_ / 2;
const float scale = 2.0f / static_cast<float>(fftSize_);
for (size_t bin = 0; bin < numBins; bin += 1) {
const float re = fftOutput_[bin].real();
const float im = fftOutput_[bin].imag();
const float magnitude = std::sqrt((re * re) + (im * im)) * scale;
magnitudesLinear_[bin] = magnitude;
magnitudesDb_[bin] = 20.0f * std::log10(std::max(magnitude, 1.0e-10f));
phases_[bin] = std::atan2(im, re);
const float leftRe = fftOutput_[bin].real();
const float leftIm = fftOutput_[bin].imag();
const float rightRe = rightFftOutput_[bin].real();
const float rightIm = rightFftOutput_[bin].imag();
const float leftMagnitude = std::sqrt((leftRe * leftRe) + (leftIm * leftIm)) * magnitudeScale_;
const float rightMagnitude = std::sqrt((rightRe * rightRe) + (rightIm * rightIm)) * magnitudeScale_;
const float stereoMagnitude = std::sqrt(
0.5f * ((leftMagnitude * leftMagnitude) + (rightMagnitude * rightMagnitude))
);
magnitudesLinear_[bin] = stereoMagnitude;
magnitudesDb_[bin] = 20.0f * std::log10(std::max(stereoMagnitude, 1.0e-10f));
phases_[bin] = std::atan2(leftIm, leftRe);
rightPhases_[bin] = std::atan2(rightIm, rightRe);
dominantRight_[bin] = rightMagnitude > leftMagnitude ? 1 : 0;
}
computeStandardSpectrum();
computeReassignedSpectrum();
blendAndShapeColumn(display, heat);
if (config_.clarityMode == "focused") {
computeFocusedSpectrum();
} else {
computeReassignedSpectrum();
}
shapeColumn(display, heat);
lastPhases_ = phases_;
rightLastPhases_ = rightPhases_;
haveLastPhase_ = true;
}
SpectrogramProcessResult SpectrogramAnalyzer::process(const float* samples, size_t length) {
return processStereo(samples, samples, length);
}
SpectrogramProcessResult SpectrogramAnalyzer::processStereo(const float* left, const float* right, size_t length) {
SpectrogramProcessResult result;
result.rowCount = config_.rowCount;
if (!samples || length == 0 || fftSize_ == 0 || config_.rowCount == 0) {
if (!left || !right || length == 0 || fftSize_ == 0 || config_.rowCount == 0) {
return result;
}
@@ -494,7 +704,8 @@ SpectrogramProcessResult SpectrogramAnalyzer::process(const float* samples, size
const size_t overlapSamples = fftSize_ - hopSize;
for (size_t index = 0; index < length; index += 1) {
frameBuffer_[frameFill_] = samples[index];
frameBuffer_[frameFill_] = left[index];
rightFrameBuffer_[frameFill_] = right[index];
frameFill_ += 1;
if (frameFill_ >= fftSize_) {
@@ -503,6 +714,7 @@ SpectrogramProcessResult SpectrogramAnalyzer::process(const float* samples, size
if (overlapSamples > 0) {
std::memmove(frameBuffer_.data(), frameBuffer_.data() + hopSize, overlapSamples * sizeof(float));
std::memmove(rightFrameBuffer_.data(), rightFrameBuffer_.data() + hopSize, overlapSamples * sizeof(float));
}
frameFill_ = overlapSamples;
}
native/src/spectrogram.h
+19 -5
View File
@@ -1,6 +1,7 @@
#pragma once
#include "dsp_utils.h"
#include <cstdint>
#include <complex>
#include <memory>
#include <string>
@@ -37,13 +38,15 @@ public:
void configure(const SpectrogramConfig& config);
SpectrogramProcessResult process(const float* samples, size_t length);
SpectrogramProcessResult processStereo(const float* left, const float* right, size_t length);
void reset();
private:
struct ClarityProfile {
float gamma;
float sharpness;
float lineWidth;
float displayShoulder;
float heatShoulder;
bool useReassignment;
};
SpectrogramConfig config_;
@@ -51,16 +54,23 @@ private:
size_t paddedSize_;
size_t frameFill_;
bool haveLastPhase_;
float magnitudeScale_;
std::unique_ptr<DSP::FFT> fft_;
std::vector<float> frameBuffer_;
std::vector<float> rightFrameBuffer_;
std::vector<float> window_;
std::vector<float> windowedInput_;
std::vector<float> rightWindowedInput_;
std::vector<std::complex<float>> fftOutput_;
std::vector<std::complex<float>> rightFftOutput_;
std::vector<float> magnitudesDb_;
std::vector<float> magnitudesLinear_;
std::vector<float> phases_;
std::vector<float> lastPhases_;
std::vector<float> rightPhases_;
std::vector<float> rightLastPhases_;
std::vector<uint8_t> dominantRight_;
std::vector<float> rowCenterBins_;
std::vector<float> rowBandStartBins_;
@@ -69,8 +79,9 @@ private:
std::vector<float> standardRaw_;
std::vector<float> standardHeat_;
std::vector<float> reassignedPower_;
std::vector<float> blendedRaw_;
std::vector<float> blendedHeat_;
std::vector<float> focusedPower_;
std::vector<float> sourceRaw_;
std::vector<float> sourceHeat_;
std::vector<float> shapedDisplay_;
std::vector<float> shapedHeat_;
std::vector<float> strokedDisplay_;
@@ -81,9 +92,12 @@ private:
void processFrame(std::vector<float>& display, std::vector<float>& heat);
void computeStandardSpectrum();
void computeReassignedSpectrum();
void blendAndShapeColumn(std::vector<float>& display, std::vector<float>& heat);
void computeFocusedSpectrum();
void shapeColumn(std::vector<float>& display, std::vector<float>& heat);
void shapeFocusedColumn(std::vector<float>& display, std::vector<float>& heat);
size_t resolveHopSize() const;
float sampleDbAtBin(float bin) const;
float samplePeakDbInBand(float startBin, float endBin, float& peakBin) const;
float frequencyFromScale(float normalizedPosition) const;
float frequencyToRow(float frequency) const;
float applyDisplayTilt(float db, float frequency) const;