LOVE/megasource
1
0
Fork 0
mirror of https://github.com/love2d/megasource.git synced 2026-08-20 04:30:45 +02:00
Code Issues Packages Projects Releases Wiki Activity

Update OpenAL-soft to 1.23.1-bc7cb17.

Browse Source
This commit is contained in:
Miku AuahDark
2024-03-20 11:06:03 +08:00
parent 4a512be715
commit 73a6fc9196
294 changed files with 44342 additions and 40077 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libs/openal-soft/core/mixer/defs.h
+31 -15
View File
@@ -2,11 +2,14 @@
#define CORE_MIXER_DEFS_H
#include <array>
#include <stdlib.h>
#include <cstdint>
#include <cstdlib>
#include <utility>
#include <variant>
#include "alspan.h"
#include "core/bufferline.h"
#include "core/resampler_limits.h"
#include "core/cubic_defs.h"
struct HrtfChannelState;
struct HrtfFilter;
@@ -16,14 +19,15 @@ using uint = unsigned int;
using float2 = std::array<float,2>;
constexpr int MixerFracBits{12};
constexpr int MixerFracOne{1 << MixerFracBits};
constexpr int MixerFracMask{MixerFracOne - 1};
inline constexpr int MixerFracBits{16};
inline constexpr int MixerFracOne{1 << MixerFracBits};
inline constexpr int MixerFracMask{MixerFracOne - 1};
inline constexpr int MixerFracHalf{MixerFracOne >> 1};
constexpr float GainSilenceThreshold{0.00001f}; /* -100dB */
inline constexpr float GainSilenceThreshold{0.00001f}; /* -100dB */
enum class Resampler {
enum class Resampler : std::uint8_t {
Point,
Linear,
Cubic,
@@ -50,23 +54,32 @@ struct BsincState {
const float *filter;
};
union InterpState {
BsincState bsinc;
struct CubicState {
/* Filter coefficients, and coefficient deltas. Starting at phase index 0,
* each subsequent phase index follows contiguously.
*/
al::span<const CubicCoefficients,CubicPhaseCount> filter;
CubicState(al::span<const CubicCoefficients,CubicPhaseCount> f) : filter{f} { }
};
using ResamplerFunc = float*(*)(const InterpState *state, float *RESTRICT src, uint frac,
uint increment, const al::span<float> dst);
using InterpState = std::variant<std::monostate,CubicState,BsincState>;
using ResamplerFunc = void(*)(const InterpState *state, const float *src, uint frac,
const uint increment, const al::span<float> dst);
ResamplerFunc PrepareResampler(Resampler resampler, uint increment, InterpState *state);
template<typename TypeTag, typename InstTag>
float *Resample_(const InterpState *state, float *RESTRICT src, uint frac, uint increment,
void Resample_(const InterpState *state, const float *src, uint frac, const uint increment,
const al::span<float> dst);
template<typename InstTag>
void Mix_(const al::span<const float> InSamples, const al::span<FloatBufferLine> OutBuffer,
float *CurrentGains, const float *TargetGains, const size_t Counter, const size_t OutPos);
template<typename InstTag>
void Mix_(const al::span<const float> InSamples, float *OutBuffer, float &CurrentGain,
const float TargetGain, const size_t Counter);
template<typename InstTag>
void MixHrtf_(const float *InSamples, float2 *AccumSamples, const uint IrSize,
@@ -77,15 +90,18 @@ void MixHrtfBlend_(const float *InSamples, float2 *AccumSamples, const uint IrSi
template<typename InstTag>
void MixDirectHrtf_(const FloatBufferSpan LeftOut, const FloatBufferSpan RightOut,
const al::span<const FloatBufferLine> InSamples, float2 *AccumSamples,
float *TempBuf, HrtfChannelState *ChanState, const size_t IrSize, const size_t BufferSize);
const al::span<float,BufferLineSize> TempBuf, HrtfChannelState *ChanState, const size_t IrSize,
const size_t BufferSize);
/* Vectorized resampler helpers */
template<size_t N>
inline void InitPosArrays(uint frac, uint increment, uint (&frac_arr)[N], uint (&pos_arr)[N])
constexpr void InitPosArrays(uint frac, const uint increment, const al::span<uint,N> frac_arr,
const al::span<uint,N> pos_arr)
{
static_assert(pos_arr.size() == frac_arr.size());
pos_arr[0] = 0;
frac_arr[0] = frac;
for(size_t i{1};i < N;i++)
for(size_t i{1};i < pos_arr.size();i++)
{
const uint frac_tmp{frac_arr[i-1] + increment};
pos_arr[i] = pos_arr[i-1] + (frac_tmp>>MixerFracBits);
libs/openal-soft/core/mixer/hrtfbase.h
+5 -4
View File
@@ -50,7 +50,7 @@ inline void MixHrtfBlendBase(const float *InSamples, float2 *RESTRICT AccumSampl
const ConstHrirSpan NewCoeffs{newparams->Coeffs};
const float newGainStep{newparams->GainStep};
if LIKELY(oldparams->Gain > GainSilenceThreshold)
if(oldparams->Gain > GainSilenceThreshold) LIKELY
{
size_t ldelay{HrtfHistoryLength - oldparams->Delay[0]};
size_t rdelay{HrtfHistoryLength - oldparams->Delay[1]};
@@ -66,7 +66,7 @@ inline void MixHrtfBlendBase(const float *InSamples, float2 *RESTRICT AccumSampl
}
}
if LIKELY(newGainStep*static_cast<float>(BufferSize) > GainSilenceThreshold)
if(newGainStep*static_cast<float>(BufferSize) > GainSilenceThreshold) LIKELY
{
size_t ldelay{HrtfHistoryLength+1 - newparams->Delay[0]};
size_t rdelay{HrtfHistoryLength+1 - newparams->Delay[1]};
@@ -86,7 +86,8 @@ inline void MixHrtfBlendBase(const float *InSamples, float2 *RESTRICT AccumSampl
template<ApplyCoeffsT ApplyCoeffs>
inline void MixDirectHrtfBase(const FloatBufferSpan LeftOut, const FloatBufferSpan RightOut,
const al::span<const FloatBufferLine> InSamples, float2 *RESTRICT AccumSamples,
float *TempBuf, HrtfChannelState *ChanState, const size_t IrSize, const size_t BufferSize)
const al::span<float,BufferLineSize> TempBuf, HrtfChannelState *ChanState, const size_t IrSize,
const size_t BufferSize)
{
ASSUME(BufferSize > 0);
@@ -100,7 +101,7 @@ inline void MixDirectHrtfBase(const FloatBufferSpan LeftOut, const FloatBufferSp
ChanState->mHfScale);
/* Now apply the HRIR coefficients to this channel. */
const float *RESTRICT tempbuf{al::assume_aligned<16>(TempBuf)};
const float *RESTRICT tempbuf{al::assume_aligned<16>(TempBuf.data())};
const ConstHrirSpan Coeffs{ChanState->mCoeffs};
for(size_t i{0u};i < BufferSize;++i)
{
libs/openal-soft/core/mixer/mixer_c.cpp
+152 -105
View File
@@ -1,16 +1,22 @@
#include "config.h"
#include <cassert>
#include <cmath>
#include <algorithm>
#include <array>
#include <cstddef>
#include <limits>
#include <variant>
#include "alnumeric.h"
#include "core/bsinc_tables.h"
#include "alspan.h"
#include "core/bsinc_defs.h"
#include "core/bufferline.h"
#include "core/cubic_defs.h"
#include "core/mixer/hrtfdefs.h"
#include "defs.h"
#include "hrtfbase.h"
#include "opthelpers.h"
struct CTag;
struct CopyTag;
struct PointTag;
struct LerpTag;
struct CubicTag;
@@ -20,73 +26,105 @@ struct FastBSincTag;
namespace {
constexpr uint FracPhaseBitDiff{MixerFracBits - BSincPhaseBits};
constexpr uint FracPhaseDiffOne{1 << FracPhaseBitDiff};
constexpr uint BsincPhaseDiffBits{MixerFracBits - BSincPhaseBits};
constexpr uint BsincPhaseDiffOne{1 << BsincPhaseDiffBits};
constexpr uint BsincPhaseDiffMask{BsincPhaseDiffOne - 1u};
inline float do_point(const InterpState&, const float *RESTRICT vals, const uint)
{ return vals[0]; }
inline float do_lerp(const InterpState&, const float *RESTRICT vals, const uint frac)
constexpr uint CubicPhaseDiffBits{MixerFracBits - CubicPhaseBits};
constexpr uint CubicPhaseDiffOne{1 << CubicPhaseDiffBits};
constexpr uint CubicPhaseDiffMask{CubicPhaseDiffOne - 1u};
constexpr
auto do_point(const float *vals, const uint) noexcept -> float { return vals[0]; }
constexpr
auto do_lerp(const float *vals, const uint frac) noexcept -> float
{ return lerpf(vals[0], vals[1], static_cast<float>(frac)*(1.0f/MixerFracOne)); }
inline float do_cubic(const InterpState&, const float *RESTRICT vals, const uint frac)
{ return cubic(vals[0], vals[1], vals[2], vals[3], static_cast<float>(frac)*(1.0f/MixerFracOne)); }
inline float do_bsinc(const InterpState &istate, const float *RESTRICT vals, const uint frac)
constexpr
auto do_cubic(const CubicState &istate, const float *vals, const uint frac) noexcept -> float
{
const size_t m{istate.bsinc.m};
/* Calculate the phase index and factor. */
const uint pi{frac >> CubicPhaseDiffBits};
const float pf{static_cast<float>(frac&CubicPhaseDiffMask) * (1.0f/CubicPhaseDiffOne)};
const auto fil = al::span{istate.filter[pi].mCoeffs};
const auto phd = al::span{istate.filter[pi].mDeltas};
/* Apply the phase interpolated filter. */
return (fil[0] + pf*phd[0])*vals[0] + (fil[1] + pf*phd[1])*vals[1]
+ (fil[2] + pf*phd[2])*vals[2] + (fil[3] + pf*phd[3])*vals[3];
}
constexpr
auto do_bsinc(const BsincState &istate, const float *vals, const uint frac) noexcept -> float
{
const size_t m{istate.m};
ASSUME(m > 0);
// Calculate the phase index and factor.
const uint pi{frac >> FracPhaseBitDiff};
const float pf{static_cast<float>(frac & (FracPhaseDiffOne-1)) * (1.0f/FracPhaseDiffOne)};
/* Calculate the phase index and factor. */
const uint pi{frac >> BsincPhaseDiffBits};
const float pf{static_cast<float>(frac&BsincPhaseDiffMask) * (1.0f/BsincPhaseDiffOne)};
const float *RESTRICT fil{istate.bsinc.filter + m*pi*2};
const float *RESTRICT phd{fil + m};
const float *RESTRICT scd{fil + BSincPhaseCount*2*m};
const float *RESTRICT spd{scd + m};
const float *fil{istate.filter + m*pi*2_uz};
const float *phd{fil + m};
const float *scd{fil + BSincPhaseCount*2_uz*m};
const float *spd{scd + m};
// Apply the scale and phase interpolated filter.
/* Apply the scale and phase interpolated filter. */
float r{0.0f};
for(size_t j_f{0};j_f < m;j_f++)
r += (fil[j_f] + istate.bsinc.sf*scd[j_f] + pf*(phd[j_f] + istate.bsinc.sf*spd[j_f])) * vals[j_f];
r += (fil[j_f] + istate.sf*scd[j_f] + pf*(phd[j_f] + istate.sf*spd[j_f])) * vals[j_f];
return r;
}
inline float do_fastbsinc(const InterpState &istate, const float *RESTRICT vals, const uint frac)
constexpr
auto do_fastbsinc(const BsincState &istate, const float *vals, const uint frac) noexcept -> float
{
const size_t m{istate.bsinc.m};
const size_t m{istate.m};
ASSUME(m > 0);
// Calculate the phase index and factor.
const uint pi{frac >> FracPhaseBitDiff};
const float pf{static_cast<float>(frac & (FracPhaseDiffOne-1)) * (1.0f/FracPhaseDiffOne)};
/* Calculate the phase index and factor. */
const uint pi{frac >> BsincPhaseDiffBits};
const float pf{static_cast<float>(frac&BsincPhaseDiffMask) * (1.0f/BsincPhaseDiffOne)};
const float *RESTRICT fil{istate.bsinc.filter + m*pi*2};
const float *RESTRICT phd{fil + m};
const float *fil{istate.filter + m*pi*2_uz};
const float *phd{fil + m};
// Apply the phase interpolated filter.
/* Apply the phase interpolated filter. */
float r{0.0f};
for(size_t j_f{0};j_f < m;j_f++)
r += (fil[j_f] + pf*phd[j_f]) * vals[j_f];
return r;
}
using SamplerT = float(&)(const InterpState&, const float*RESTRICT, const uint);
template<SamplerT Sampler>
float *DoResample(const InterpState *state, float *RESTRICT src, uint frac, uint increment,
const al::span<float> dst)
template<float(&Sampler)(const float*, const uint)noexcept>
void DoResample(const float *src, uint frac, const uint increment, const al::span<float> dst)
{
const InterpState istate{*state};
for(float &out : dst)
ASSUME(frac < MixerFracOne);
std::generate(dst.begin(), dst.end(), [&src,&frac,increment]() -> float
{
out = Sampler(istate, src, frac);
const float output{Sampler(src, frac)};
frac += increment;
src += frac>>MixerFracBits;
frac &= MixerFracMask;
}
return dst.data();
return output;
});
}
inline void ApplyCoeffs(float2 *RESTRICT Values, const size_t IrSize, const ConstHrirSpan Coeffs,
const float left, const float right)
template<typename U, float(&Sampler)(const U&, const float*,const uint)noexcept>
void DoResample(const U istate, const float *src, uint frac, const uint increment,
const al::span<float> dst)
{
ASSUME(frac < MixerFracOne);
std::generate(dst.begin(), dst.end(), [istate,&src,&frac,increment]() -> float
{
const float output{Sampler(istate, src, frac)};
frac += increment;
src += frac>>MixerFracBits;
frac &= MixerFracMask;
return output;
});
}
constexpr void ApplyCoeffs(float2 *RESTRICT Values, const size_t IrSize,
const ConstHrirSpan Coeffs, const float left, const float right) noexcept
{
ASSUME(IrSize >= MinIrLength);
for(size_t c{0};c < IrSize;++c)
@@ -96,45 +134,69 @@ inline void ApplyCoeffs(float2 *RESTRICT Values, const size_t IrSize, const Cons
}
}
force_inline void MixLine(const al::span<const float> InSamples, float *RESTRICT dst,
float &CurrentGain, const float TargetGain, const float delta, const size_t min_len,
size_t Counter)
{
float gain{CurrentGain};
const float step{(TargetGain-gain) * delta};
size_t pos{0};
if(!(std::abs(step) > std::numeric_limits<float>::epsilon()))
gain = TargetGain;
else
{
float step_count{0.0f};
for(;pos != min_len;++pos)
{
dst[pos] += InSamples[pos] * (gain + step*step_count);
step_count += 1.0f;
}
if(pos == Counter)
gain = TargetGain;
else
gain += step*step_count;
}
CurrentGain = gain;
if(!(std::abs(gain) > GainSilenceThreshold))
return;
for(;pos != InSamples.size();++pos)
dst[pos] += InSamples[pos] * gain;
}
} // namespace
template<>
float *Resample_<CopyTag,CTag>(const InterpState*, float *RESTRICT src, uint, uint,
void Resample_<PointTag,CTag>(const InterpState*, const float *src, uint frac,
const uint increment, const al::span<float> dst)
{ DoResample<do_point>(src, frac, increment, dst); }
template<>
void Resample_<LerpTag,CTag>(const InterpState*, const float *src, uint frac, const uint increment,
const al::span<float> dst)
{ DoResample<do_lerp>(src, frac, increment, dst); }
template<>
void Resample_<CubicTag,CTag>(const InterpState *state, const float *src, uint frac,
const uint increment, const al::span<float> dst)
{ DoResample<CubicState,do_cubic>(std::get<CubicState>(*state), src-1, frac, increment, dst); }
template<>
void Resample_<BSincTag,CTag>(const InterpState *state, const float *src, uint frac,
const uint increment, const al::span<float> dst)
{
#if defined(HAVE_SSE) || defined(HAVE_NEON)
/* Avoid copying the source data if it's aligned like the destination. */
if((reinterpret_cast<intptr_t>(src)&15) == (reinterpret_cast<intptr_t>(dst.data())&15))
return src;
#endif
std::copy_n(src, dst.size(), dst.begin());
return dst.data();
const auto istate = std::get<BsincState>(*state);
DoResample<BsincState,do_bsinc>(istate, src-istate.l, frac, increment, dst);
}
template<>
float *Resample_<PointTag,CTag>(const InterpState *state, float *RESTRICT src, uint frac,
uint increment, const al::span<float> dst)
{ return DoResample<do_point>(state, src, frac, increment, dst); }
template<>
float *Resample_<LerpTag,CTag>(const InterpState *state, float *RESTRICT src, uint frac,
uint increment, const al::span<float> dst)
{ return DoResample<do_lerp>(state, src, frac, increment, dst); }
template<>
float *Resample_<CubicTag,CTag>(const InterpState *state, float *RESTRICT src, uint frac,
uint increment, const al::span<float> dst)
{ return DoResample<do_cubic>(state, src-1, frac, increment, dst); }
template<>
float *Resample_<BSincTag,CTag>(const InterpState *state, float *RESTRICT src, uint frac,
uint increment, const al::span<float> dst)
{ return DoResample<do_bsinc>(state, src-state->bsinc.l, frac, increment, dst); }
template<>
float *Resample_<FastBSincTag,CTag>(const InterpState *state, float *RESTRICT src, uint frac,
uint increment, const al::span<float> dst)
{ return DoResample<do_fastbsinc>(state, src-state->bsinc.l, frac, increment, dst); }
void Resample_<FastBSincTag,CTag>(const InterpState *state, const float *src, uint frac,
const uint increment, const al::span<float> dst)
{
const auto istate = std::get<BsincState>(*state);
DoResample<BsincState,do_fastbsinc>(istate, src-istate.l, frac, increment, dst);
}
template<>
@@ -153,7 +215,8 @@ void MixHrtfBlend_<CTag>(const float *InSamples, float2 *AccumSamples, const uin
template<>
void MixDirectHrtf_<CTag>(const FloatBufferSpan LeftOut, const FloatBufferSpan RightOut,
const al::span<const FloatBufferLine> InSamples, float2 *AccumSamples,
float *TempBuf, HrtfChannelState *ChanState, const size_t IrSize, const size_t BufferSize)
const al::span<float,BufferLineSize> TempBuf, HrtfChannelState *ChanState, const size_t IrSize,
const size_t BufferSize)
{
MixDirectHrtfBase<ApplyCoeffs>(LeftOut, RightOut, InSamples, AccumSamples, TempBuf, ChanState,
IrSize, BufferSize);
@@ -165,36 +228,20 @@ void Mix_<CTag>(const al::span<const float> InSamples, const al::span<FloatBuffe
float *CurrentGains, const float *TargetGains, const size_t Counter, const size_t OutPos)
{
const float delta{(Counter > 0) ? 1.0f / static_cast<float>(Counter) : 0.0f};
const auto min_len = minz(Counter, InSamples.size());
const auto min_len = std::min(Counter, InSamples.size());
for(FloatBufferLine &output : OutBuffer)
{
float *RESTRICT dst{al::assume_aligned<16>(output.data()+OutPos)};
float gain{*CurrentGains};
const float step{(*TargetGains-gain) * delta};
size_t pos{0};
if(!(std::abs(step) > std::numeric_limits<float>::epsilon()))
gain = *TargetGains;
else
{
float step_count{0.0f};
for(;pos != min_len;++pos)
{
dst[pos] += InSamples[pos] * (gain + step*step_count);
step_count += 1.0f;
}
if(pos == Counter)
gain = *TargetGains;
else
gain += step*step_count;
}
*CurrentGains = gain;
++CurrentGains;
++TargetGains;
if(!(std::abs(gain) > GainSilenceThreshold))
continue;
for(;pos != InSamples.size();++pos)
dst[pos] += InSamples[pos] * gain;
}
MixLine(InSamples, al::assume_aligned<16>(output.data()+OutPos), *CurrentGains++,
*TargetGains++, delta, min_len, Counter);
}
template<>
void Mix_<CTag>(const al::span<const float> InSamples, float *OutBuffer, float &CurrentGain,
const float TargetGain, const size_t Counter)
{
const float delta{(Counter > 0) ? 1.0f / static_cast<float>(Counter) : 0.0f};
const auto min_len = std::min(Counter, InSamples.size());
MixLine(InSamples, al::assume_aligned<16>(OutBuffer), CurrentGain,
TargetGain, delta, min_len, Counter);
}
libs/openal-soft/core/mixer/mixer_neon.cpp
+280 -143
View File
@@ -2,16 +2,25 @@
#include <arm_neon.h>
#include <cmath>
#include <algorithm>
#include <array>
#include <cstddef>
#include <limits>
#include <variant>
#include "alnumeric.h"
#include "alspan.h"
#include "core/bsinc_defs.h"
#include "core/bufferline.h"
#include "core/cubic_defs.h"
#include "core/mixer/hrtfdefs.h"
#include "defs.h"
#include "hrtfbase.h"
#include "opthelpers.h"
struct NEONTag;
struct LerpTag;
struct CubicTag;
struct BSincTag;
struct FastBSincTag;
@@ -20,8 +29,31 @@ struct FastBSincTag;
#pragma GCC target("fpu=neon")
#endif
using uint = unsigned int;
namespace {
constexpr uint BSincPhaseDiffBits{MixerFracBits - BSincPhaseBits};
constexpr uint BSincPhaseDiffOne{1 << BSincPhaseDiffBits};
constexpr uint BSincPhaseDiffMask{BSincPhaseDiffOne - 1u};
constexpr uint CubicPhaseDiffBits{MixerFracBits - CubicPhaseBits};
constexpr uint CubicPhaseDiffOne{1 << CubicPhaseDiffBits};
constexpr uint CubicPhaseDiffMask{CubicPhaseDiffOne - 1u};
force_inline
void vtranspose4(float32x4_t &x0, float32x4_t &x1, float32x4_t &x2, float32x4_t &x3) noexcept
{
float32x4x2_t t0_{vzipq_f32(x0, x2)};
float32x4x2_t t1_{vzipq_f32(x1, x3)};
float32x4x2_t u0_{vzipq_f32(t0_.val[0], t1_.val[0])};
float32x4x2_t u1_{vzipq_f32(t0_.val[1], t1_.val[1])};
x0 = u0_.val[0];
x1 = u0_.val[1];
x2 = u1_.val[0];
x3 = u1_.val[1];
}
inline float32x4_t set_f4(float l0, float l1, float l2, float l3)
{
float32x4_t ret{vmovq_n_f32(l0)};
@@ -31,18 +63,15 @@ inline float32x4_t set_f4(float l0, float l1, float l2, float l3)
return ret;
}
constexpr uint FracPhaseBitDiff{MixerFracBits - BSincPhaseBits};
constexpr uint FracPhaseDiffOne{1 << FracPhaseBitDiff};
inline void ApplyCoeffs(float2 *RESTRICT Values, const size_t IrSize, const ConstHrirSpan Coeffs,
const float left, const float right)
{
float32x4_t leftright4;
auto dup_samples = [left,right]
{
float32x2_t leftright2{vmov_n_f32(left)};
leftright2 = vset_lane_f32(right, leftright2, 1);
leftright4 = vcombine_f32(leftright2, leftright2);
}
float32x2_t leftright2{vset_lane_f32(right, vmov_n_f32(left), 1)};
return vcombine_f32(leftright2, leftright2);
};
const float32x4_t leftright4{dup_samples()};
ASSUME(IrSize >= MinIrLength);
for(size_t c{0};c < IrSize;c += 2)
@@ -56,85 +85,247 @@ inline void ApplyCoeffs(float2 *RESTRICT Values, const size_t IrSize, const Cons
}
}
force_inline void MixLine(const al::span<const float> InSamples, float *RESTRICT dst,
float &CurrentGain, const float TargetGain, const float delta, const size_t min_len,
const size_t aligned_len, size_t Counter)
{
float gain{CurrentGain};
const float step{(TargetGain-gain) * delta};
size_t pos{0};
if(!(std::abs(step) > std::numeric_limits<float>::epsilon()))
gain = TargetGain;
else
{
float step_count{0.0f};
/* Mix with applying gain steps in aligned multiples of 4. */
if(size_t todo{min_len >> 2})
{
const float32x4_t four4{vdupq_n_f32(4.0f)};
const float32x4_t step4{vdupq_n_f32(step)};
const float32x4_t gain4{vdupq_n_f32(gain)};
float32x4_t step_count4{vdupq_n_f32(0.0f)};
step_count4 = vsetq_lane_f32(1.0f, step_count4, 1);
step_count4 = vsetq_lane_f32(2.0f, step_count4, 2);
step_count4 = vsetq_lane_f32(3.0f, step_count4, 3);
do {
const float32x4_t val4 = vld1q_f32(&InSamples[pos]);
float32x4_t dry4 = vld1q_f32(&dst[pos]);
dry4 = vmlaq_f32(dry4, val4, vmlaq_f32(gain4, step4, step_count4));
step_count4 = vaddq_f32(step_count4, four4);
vst1q_f32(&dst[pos], dry4);
pos += 4;
} while(--todo);
/* NOTE: step_count4 now represents the next four counts after the
* last four mixed samples, so the lowest element represents the
* next step count to apply.
*/
step_count = vgetq_lane_f32(step_count4, 0);
}
/* Mix with applying left over gain steps that aren't aligned multiples of 4. */
for(size_t leftover{min_len&3};leftover;++pos,--leftover)
{
dst[pos] += InSamples[pos] * (gain + step*step_count);
step_count += 1.0f;
}
if(pos == Counter)
gain = TargetGain;
else
gain += step*step_count;
/* Mix until pos is aligned with 4 or the mix is done. */
for(size_t leftover{aligned_len&3};leftover;++pos,--leftover)
dst[pos] += InSamples[pos] * gain;
}
CurrentGain = gain;
if(!(std::abs(gain) > GainSilenceThreshold))
return;
if(size_t todo{(InSamples.size()-pos) >> 2})
{
const float32x4_t gain4 = vdupq_n_f32(gain);
do {
const float32x4_t val4 = vld1q_f32(&InSamples[pos]);
float32x4_t dry4 = vld1q_f32(&dst[pos]);
dry4 = vmlaq_f32(dry4, val4, gain4);
vst1q_f32(&dst[pos], dry4);
pos += 4;
} while(--todo);
}
for(size_t leftover{(InSamples.size()-pos)&3};leftover;++pos,--leftover)
dst[pos] += InSamples[pos] * gain;
}
} // namespace
template<>
float *Resample_<LerpTag,NEONTag>(const InterpState*, float *RESTRICT src, uint frac,
uint increment, const al::span<float> dst)
void Resample_<LerpTag,NEONTag>(const InterpState*, const float *src, uint frac,
const uint increment, const al::span<float> dst)
{
const int32x4_t increment4 = vdupq_n_s32(static_cast<int>(increment*4));
ASSUME(frac < MixerFracOne);
const uint32x4_t increment4 = vdupq_n_u32(increment*4u);
const float32x4_t fracOne4 = vdupq_n_f32(1.0f/MixerFracOne);
const int32x4_t fracMask4 = vdupq_n_s32(MixerFracMask);
alignas(16) uint pos_[4], frac_[4];
int32x4_t pos4, frac4;
const uint32x4_t fracMask4 = vdupq_n_u32(MixerFracMask);
InitPosArrays(frac, increment, frac_, pos_);
frac4 = vld1q_s32(reinterpret_cast<int*>(frac_));
pos4 = vld1q_s32(reinterpret_cast<int*>(pos_));
alignas(16) std::array<uint,4> pos_, frac_;
InitPosArrays(frac, increment, al::span{frac_}, al::span{pos_});
uint32x4_t frac4 = vld1q_u32(frac_.data());
uint32x4_t pos4 = vld1q_u32(pos_.data());
auto dst_iter = dst.begin();
for(size_t todo{dst.size()>>2};todo;--todo)
auto vecout = al::span<float32x4_t>{reinterpret_cast<float32x4_t*>(dst.data()), dst.size()/4};
std::generate(vecout.begin(), vecout.end(), [=,&pos4,&frac4]() -> float32x4_t
{
const int pos0{vgetq_lane_s32(pos4, 0)};
const int pos1{vgetq_lane_s32(pos4, 1)};
const int pos2{vgetq_lane_s32(pos4, 2)};
const int pos3{vgetq_lane_s32(pos4, 3)};
const uint pos0{vgetq_lane_u32(pos4, 0)};
const uint pos1{vgetq_lane_u32(pos4, 1)};
const uint pos2{vgetq_lane_u32(pos4, 2)};
const uint pos3{vgetq_lane_u32(pos4, 3)};
const float32x4_t val1{set_f4(src[pos0], src[pos1], src[pos2], src[pos3])};
const float32x4_t val2{set_f4(src[pos0+1], src[pos1+1], src[pos2+1], src[pos3+1])};
const float32x4_t val2{set_f4(src[pos0+1_uz], src[pos1+1_uz], src[pos2+1_uz], src[pos3+1_uz])};
/* val1 + (val2-val1)*mu */
const float32x4_t r0{vsubq_f32(val2, val1)};
const float32x4_t mu{vmulq_f32(vcvtq_f32_s32(frac4), fracOne4)};
const float32x4_t mu{vmulq_f32(vcvtq_f32_u32(frac4), fracOne4)};
const float32x4_t out{vmlaq_f32(val1, mu, r0)};
vst1q_f32(dst_iter, out);
dst_iter += 4;
frac4 = vaddq_s32(frac4, increment4);
pos4 = vaddq_s32(pos4, vshrq_n_s32(frac4, MixerFracBits));
frac4 = vandq_s32(frac4, fracMask4);
}
frac4 = vaddq_u32(frac4, increment4);
pos4 = vaddq_u32(pos4, vshrq_n_u32(frac4, MixerFracBits));
frac4 = vandq_u32(frac4, fracMask4);
return out;
});
if(size_t todo{dst.size()&3})
{
src += static_cast<uint>(vgetq_lane_s32(pos4, 0));
frac = static_cast<uint>(vgetq_lane_s32(frac4, 0));
src += vgetq_lane_u32(pos4, 0);
frac = vgetq_lane_u32(frac4, 0);
do {
*(dst_iter++) = lerpf(src[0], src[1], static_cast<float>(frac) * (1.0f/MixerFracOne));
std::generate(dst.end()-ptrdiff_t(todo), dst.end(), [&src,&frac,increment]
{
const float out{lerpf(src[0], src[1], static_cast<float>(frac) * (1.0f/MixerFracOne))};
frac += increment;
src += frac>>MixerFracBits;
frac &= MixerFracMask;
} while(--todo);
return out;
});
}
return dst.data();
}
template<>
float *Resample_<BSincTag,NEONTag>(const InterpState *state, float *RESTRICT src, uint frac,
uint increment, const al::span<float> dst)
void Resample_<CubicTag,NEONTag>(const InterpState *state, const float *src, uint frac,
const uint increment, const al::span<float> dst)
{
const float *const filter{state->bsinc.filter};
const float32x4_t sf4{vdupq_n_f32(state->bsinc.sf)};
const size_t m{state->bsinc.m};
ASSUME(m > 0);
ASSUME(frac < MixerFracOne);
src -= state->bsinc.l;
for(float &out_sample : dst)
const auto filter = std::get<CubicState>(*state).filter;
const uint32x4_t increment4{vdupq_n_u32(increment*4u)};
const uint32x4_t fracMask4{vdupq_n_u32(MixerFracMask)};
const float32x4_t fracDiffOne4{vdupq_n_f32(1.0f/CubicPhaseDiffOne)};
const uint32x4_t fracDiffMask4{vdupq_n_u32(CubicPhaseDiffMask)};
alignas(16) std::array<uint,4> pos_, frac_;
InitPosArrays(frac, increment, al::span{frac_}, al::span{pos_});
uint32x4_t frac4{vld1q_u32(frac_.data())};
uint32x4_t pos4{vld1q_u32(pos_.data())};
src -= 1;
auto vecout = al::span<float32x4_t>{reinterpret_cast<float32x4_t*>(dst.data()), dst.size()/4};
std::generate(vecout.begin(), vecout.end(), [=,&pos4,&frac4]() -> float32x4_t
{
const uint pos0{vgetq_lane_u32(pos4, 0)};
const uint pos1{vgetq_lane_u32(pos4, 1)};
const uint pos2{vgetq_lane_u32(pos4, 2)};
const uint pos3{vgetq_lane_u32(pos4, 3)};
const float32x4_t val0{vld1q_f32(src+pos0)};
const float32x4_t val1{vld1q_f32(src+pos1)};
const float32x4_t val2{vld1q_f32(src+pos2)};
const float32x4_t val3{vld1q_f32(src+pos3)};
const uint32x4_t pi4{vshrq_n_u32(frac4, CubicPhaseDiffBits)};
const uint pi0{vgetq_lane_u32(pi4, 0)}; ASSUME(pi0 < CubicPhaseCount);
const uint pi1{vgetq_lane_u32(pi4, 1)}; ASSUME(pi1 < CubicPhaseCount);
const uint pi2{vgetq_lane_u32(pi4, 2)}; ASSUME(pi2 < CubicPhaseCount);
const uint pi3{vgetq_lane_u32(pi4, 3)}; ASSUME(pi3 < CubicPhaseCount);
const float32x4_t pf4{vmulq_f32(vcvtq_f32_u32(vandq_u32(frac4, fracDiffMask4)),
fracDiffOne4)};
float32x4_t r0{vmulq_f32(val0,
vmlaq_f32(vld1q_f32(filter[pi0].mCoeffs.data()), vdupq_lane_f32(vget_low_f32(pf4), 0),
vld1q_f32(filter[pi0].mDeltas.data())))};
float32x4_t r1{vmulq_f32(val1,
vmlaq_f32(vld1q_f32(filter[pi1].mCoeffs.data()), vdupq_lane_f32(vget_low_f32(pf4), 1),
vld1q_f32(filter[pi1].mDeltas.data())))};
float32x4_t r2{vmulq_f32(val2,
vmlaq_f32(vld1q_f32(filter[pi2].mCoeffs.data()), vdupq_lane_f32(vget_high_f32(pf4), 0),
vld1q_f32(filter[pi2].mDeltas.data())))};
float32x4_t r3{vmulq_f32(val3,
vmlaq_f32(vld1q_f32(filter[pi3].mCoeffs.data()), vdupq_lane_f32(vget_high_f32(pf4), 1),
vld1q_f32(filter[pi3].mDeltas.data())))};
vtranspose4(r0, r1, r2, r3);
r0 = vaddq_f32(vaddq_f32(r0, r1), vaddq_f32(r2, r3));
frac4 = vaddq_u32(frac4, increment4);
pos4 = vaddq_u32(pos4, vshrq_n_u32(frac4, MixerFracBits));
frac4 = vandq_u32(frac4, fracMask4);
return r0;
});
if(const size_t todo{dst.size()&3})
{
src += vgetq_lane_u32(pos4, 0);
frac = vgetq_lane_u32(frac4, 0);
std::generate(dst.end()-ptrdiff_t(todo), dst.end(), [&src,&frac,increment,filter]
{
const uint pi{frac >> CubicPhaseDiffBits}; ASSUME(pi < CubicPhaseCount);
const float pf{static_cast<float>(frac&CubicPhaseDiffMask) * (1.0f/CubicPhaseDiffOne)};
const float32x4_t pf4{vdupq_n_f32(pf)};
const float32x4_t f4{vmlaq_f32(vld1q_f32(filter[pi].mCoeffs.data()), pf4,
vld1q_f32(filter[pi].mDeltas.data()))};
float32x4_t r4{vmulq_f32(f4, vld1q_f32(src))};
r4 = vaddq_f32(r4, vrev64q_f32(r4));
const float output{vget_lane_f32(vadd_f32(vget_low_f32(r4), vget_high_f32(r4)), 0)};
frac += increment;
src += frac>>MixerFracBits;
frac &= MixerFracMask;
return output;
});
}
}
template<>
void Resample_<BSincTag,NEONTag>(const InterpState *state, const float *src, uint frac,
const uint increment, const al::span<float> dst)
{
const auto &bsinc = std::get<BsincState>(*state);
const float *const filter{bsinc.filter};
const float32x4_t sf4{vdupq_n_f32(bsinc.sf)};
const size_t m{bsinc.m};
ASSUME(m > 0);
ASSUME(frac < MixerFracOne);
src -= bsinc.l;
std::generate(dst.begin(), dst.end(), [&src,&frac,increment,filter,sf4,m]() -> float
{
// Calculate the phase index and factor.
const uint pi{frac >> FracPhaseBitDiff};
const float pf{static_cast<float>(frac & (FracPhaseDiffOne-1)) * (1.0f/FracPhaseDiffOne)};
const uint pi{frac >> BSincPhaseDiffBits};
const float pf{static_cast<float>(frac&BSincPhaseDiffMask) * (1.0f/BSincPhaseDiffOne)};
// Apply the scale and phase interpolated filter.
float32x4_t r4{vdupq_n_f32(0.0f)};
{
const float32x4_t pf4{vdupq_n_f32(pf)};
const float *RESTRICT fil{filter + m*pi*2};
const float *RESTRICT phd{fil + m};
const float *RESTRICT scd{fil + BSincPhaseCount*2*m};
const float *RESTRICT spd{scd + m};
const float *fil{filter + m*pi*2_uz};
const float *phd{fil + m};
const float *scd{fil + BSincPhaseCount*2_uz*m};
const float *spd{scd + m};
size_t td{m >> 2};
size_t j{0u};
@@ -149,36 +340,38 @@ float *Resample_<BSincTag,NEONTag>(const InterpState *state, float *RESTRICT src
} while(--td);
}
r4 = vaddq_f32(r4, vrev64q_f32(r4));
out_sample = vget_lane_f32(vadd_f32(vget_low_f32(r4), vget_high_f32(r4)), 0);
const float output{vget_lane_f32(vadd_f32(vget_low_f32(r4), vget_high_f32(r4)), 0)};
frac += increment;
src += frac>>MixerFracBits;
frac &= MixerFracMask;
}
return dst.data();
return output;
});
}
template<>
float *Resample_<FastBSincTag,NEONTag>(const InterpState *state, float *RESTRICT src, uint frac,
uint increment, const al::span<float> dst)
void Resample_<FastBSincTag,NEONTag>(const InterpState *state, const float *src,
uint frac, const uint increment, const al::span<float> dst)
{
const float *const filter{state->bsinc.filter};
const size_t m{state->bsinc.m};
const auto &bsinc = std::get<BsincState>(*state);
const float *const filter{bsinc.filter};
const size_t m{bsinc.m};
ASSUME(m > 0);
ASSUME(frac < MixerFracOne);
src -= state->bsinc.l;
for(float &out_sample : dst)
src -= bsinc.l;
std::generate(dst.begin(), dst.end(), [&src,&frac,increment,filter,m]() -> float
{
// Calculate the phase index and factor.
const uint pi{frac >> FracPhaseBitDiff};
const float pf{static_cast<float>(frac & (FracPhaseDiffOne-1)) * (1.0f/FracPhaseDiffOne)};
const uint pi{frac >> BSincPhaseDiffBits};
const float pf{static_cast<float>(frac&BSincPhaseDiffMask) * (1.0f/BSincPhaseDiffOne)};
// Apply the phase interpolated filter.
float32x4_t r4{vdupq_n_f32(0.0f)};
{
const float32x4_t pf4{vdupq_n_f32(pf)};
const float *RESTRICT fil{filter + m*pi*2};
const float *RESTRICT phd{fil + m};
const float *fil{filter + m*pi*2_uz};
const float *phd{fil + m};
size_t td{m >> 2};
size_t j{0u};
@@ -191,13 +384,13 @@ float *Resample_<FastBSincTag,NEONTag>(const InterpState *state, float *RESTRICT
} while(--td);
}
r4 = vaddq_f32(r4, vrev64q_f32(r4));
out_sample = vget_lane_f32(vadd_f32(vget_low_f32(r4), vget_high_f32(r4)), 0);
const float output{vget_lane_f32(vadd_f32(vget_low_f32(r4), vget_high_f32(r4)), 0)};
frac += increment;
src += frac>>MixerFracBits;
frac &= MixerFracMask;
}
return dst.data();
return output;
});
}
@@ -217,7 +410,8 @@ void MixHrtfBlend_<NEONTag>(const float *InSamples, float2 *AccumSamples, const
template<>
void MixDirectHrtf_<NEONTag>(const FloatBufferSpan LeftOut, const FloatBufferSpan RightOut,
const al::span<const FloatBufferLine> InSamples, float2 *AccumSamples,
float *TempBuf, HrtfChannelState *ChanState, const size_t IrSize, const size_t BufferSize)
const al::span<float,BufferLineSize> TempBuf, HrtfChannelState *ChanState, const size_t IrSize,
const size_t BufferSize)
{
MixDirectHrtfBase<ApplyCoeffs>(LeftOut, RightOut, InSamples, AccumSamples, TempBuf, ChanState,
IrSize, BufferSize);
@@ -229,79 +423,22 @@ void Mix_<NEONTag>(const al::span<const float> InSamples, const al::span<FloatBu
float *CurrentGains, const float *TargetGains, const size_t Counter, const size_t OutPos)
{
const float delta{(Counter > 0) ? 1.0f / static_cast<float>(Counter) : 0.0f};
const auto min_len = minz(Counter, InSamples.size());
const auto aligned_len = minz((min_len+3) & ~size_t{3}, InSamples.size()) - min_len;
const auto min_len = std::min(Counter, InSamples.size());
const auto aligned_len = std::min((min_len+3_uz) & ~3_uz, InSamples.size()) - min_len;
for(FloatBufferLine &output : OutBuffer)
{
float *RESTRICT dst{al::assume_aligned<16>(output.data()+OutPos)};
float gain{*CurrentGains};
const float step{(*TargetGains-gain) * delta};
size_t pos{0};
if(!(std::abs(step) > std::numeric_limits<float>::epsilon()))
gain = *TargetGains;
else
{
float step_count{0.0f};
/* Mix with applying gain steps in aligned multiples of 4. */
if(size_t todo{min_len >> 2})
{
const float32x4_t four4{vdupq_n_f32(4.0f)};
const float32x4_t step4{vdupq_n_f32(step)};
const float32x4_t gain4{vdupq_n_f32(gain)};
float32x4_t step_count4{vdupq_n_f32(0.0f)};
step_count4 = vsetq_lane_f32(1.0f, step_count4, 1);
step_count4 = vsetq_lane_f32(2.0f, step_count4, 2);
step_count4 = vsetq_lane_f32(3.0f, step_count4, 3);
do {
const float32x4_t val4 = vld1q_f32(&InSamples[pos]);
float32x4_t dry4 = vld1q_f32(&dst[pos]);
dry4 = vmlaq_f32(dry4, val4, vmlaq_f32(gain4, step4, step_count4));
step_count4 = vaddq_f32(step_count4, four4);
vst1q_f32(&dst[pos], dry4);
pos += 4;
} while(--todo);
/* NOTE: step_count4 now represents the next four counts after
* the last four mixed samples, so the lowest element
* represents the next step count to apply.
*/
step_count = vgetq_lane_f32(step_count4, 0);
}
/* Mix with applying left over gain steps that aren't aligned multiples of 4. */
for(size_t leftover{min_len&3};leftover;++pos,--leftover)
{
dst[pos] += InSamples[pos] * (gain + step*step_count);
step_count += 1.0f;
}
if(pos == Counter)
gain = *TargetGains;
else
gain += step*step_count;
/* Mix until pos is aligned with 4 or the mix is done. */
for(size_t leftover{aligned_len&3};leftover;++pos,--leftover)
dst[pos] += InSamples[pos] * gain;
}
*CurrentGains = gain;
++CurrentGains;
++TargetGains;
if(!(std::abs(gain) > GainSilenceThreshold))
continue;
if(size_t todo{(InSamples.size()-pos) >> 2})
{
const float32x4_t gain4 = vdupq_n_f32(gain);
do {
const float32x4_t val4 = vld1q_f32(&InSamples[pos]);
float32x4_t dry4 = vld1q_f32(&dst[pos]);
dry4 = vmlaq_f32(dry4, val4, gain4);
vst1q_f32(&dst[pos], dry4);
pos += 4;
} while(--todo);
}
for(size_t leftover{(InSamples.size()-pos)&3};leftover;++pos,--leftover)
dst[pos] += InSamples[pos] * gain;
}
MixLine(InSamples, al::assume_aligned<16>(output.data()+OutPos), *CurrentGains++,
*TargetGains++, delta, min_len, aligned_len, Counter);
}
template<>
void Mix_<NEONTag>(const al::span<const float> InSamples, float *OutBuffer, float &CurrentGain,
const float TargetGain, const size_t Counter)
{
const float delta{(Counter > 0) ? 1.0f / static_cast<float>(Counter) : 0.0f};
const auto min_len = std::min(Counter, InSamples.size());
const auto aligned_len = std::min((min_len+3_uz) & ~3_uz, InSamples.size()) - min_len;
MixLine(InSamples, al::assume_aligned<16>(OutBuffer), CurrentGain, TargetGain, delta, min_len,
aligned_len, Counter);
}
libs/openal-soft/core/mixer/mixer_sse.cpp
+196 -125
View File
@@ -1,16 +1,27 @@
#include "config.h"
#include <mmintrin.h>
#include <xmmintrin.h>
#include <cmath>
#include <algorithm>
#include <array>
#include <cstddef>
#include <cstdint>
#include <limits>
#include <variant>
#include "alnumeric.h"
#include "alspan.h"
#include "core/bsinc_defs.h"
#include "core/bufferline.h"
#include "core/cubic_defs.h"
#include "core/mixer/hrtfdefs.h"
#include "defs.h"
#include "hrtfbase.h"
#include "opthelpers.h"
struct SSETag;
struct CubicTag;
struct BSincTag;
struct FastBSincTag;
@@ -21,10 +32,16 @@ struct FastBSincTag;
namespace {
constexpr uint FracPhaseBitDiff{MixerFracBits - BSincPhaseBits};
constexpr uint FracPhaseDiffOne{1 << FracPhaseBitDiff};
constexpr uint BSincPhaseDiffBits{MixerFracBits - BSincPhaseBits};
constexpr uint BSincPhaseDiffOne{1 << BSincPhaseDiffBits};
constexpr uint BSincPhaseDiffMask{BSincPhaseDiffOne - 1u};
#define MLA4(x, y, z) _mm_add_ps(x, _mm_mul_ps(y, z))
constexpr uint CubicPhaseDiffBits{MixerFracBits - CubicPhaseBits};
constexpr uint CubicPhaseDiffOne{1 << CubicPhaseDiffBits};
constexpr uint CubicPhaseDiffMask{CubicPhaseDiffOne - 1u};
force_inline __m128 vmadd(const __m128 x, const __m128 y, const __m128 z) noexcept
{ return _mm_add_ps(x, _mm_mul_ps(y, z)); }
inline void ApplyCoeffs(float2 *RESTRICT Values, const size_t IrSize, const ConstHrirSpan Coeffs,
const float left, const float right)
@@ -40,130 +57,239 @@ inline void ApplyCoeffs(float2 *RESTRICT Values, const size_t IrSize, const Cons
{
for(size_t i{0};i < IrSize;i += 2)
{
const __m128 coeffs{_mm_load_ps(&Coeffs[i][0])};
__m128 vals{_mm_load_ps(&Values[i][0])};
vals = MLA4(vals, lrlr, coeffs);
_mm_store_ps(&Values[i][0], vals);
const __m128 coeffs{_mm_load_ps(Coeffs[i].data())};
__m128 vals{_mm_load_ps(Values[i].data())};
vals = vmadd(vals, lrlr, coeffs);
_mm_store_ps(Values[i].data(), vals);
}
}
else
{
__m128 imp0, imp1;
__m128 coeffs{_mm_load_ps(&Coeffs[0][0])};
__m128 vals{_mm_loadl_pi(_mm_setzero_ps(), reinterpret_cast<__m64*>(&Values[0][0]))};
__m128 coeffs{_mm_load_ps(Coeffs[0].data())};
__m128 vals{_mm_loadl_pi(_mm_setzero_ps(), reinterpret_cast<__m64*>(Values[0].data()))};
imp0 = _mm_mul_ps(lrlr, coeffs);
vals = _mm_add_ps(imp0, vals);
_mm_storel_pi(reinterpret_cast<__m64*>(&Values[0][0]), vals);
_mm_storel_pi(reinterpret_cast<__m64*>(Values[0].data()), vals);
size_t td{((IrSize+1)>>1) - 1};
size_t i{1};
do {
coeffs = _mm_load_ps(&Coeffs[i+1][0]);
vals = _mm_load_ps(&Values[i][0]);
coeffs = _mm_load_ps(Coeffs[i+1].data());
vals = _mm_load_ps(Values[i].data());
imp1 = _mm_mul_ps(lrlr, coeffs);
imp0 = _mm_shuffle_ps(imp0, imp1, _MM_SHUFFLE(1, 0, 3, 2));
vals = _mm_add_ps(imp0, vals);
_mm_store_ps(&Values[i][0], vals);
_mm_store_ps(Values[i].data(), vals);
imp0 = imp1;
i += 2;
} while(--td);
vals = _mm_loadl_pi(vals, reinterpret_cast<__m64*>(&Values[i][0]));
vals = _mm_loadl_pi(vals, reinterpret_cast<__m64*>(Values[i].data()));
imp0 = _mm_movehl_ps(imp0, imp0);
vals = _mm_add_ps(imp0, vals);
_mm_storel_pi(reinterpret_cast<__m64*>(&Values[i][0]), vals);
_mm_storel_pi(reinterpret_cast<__m64*>(Values[i].data()), vals);
}
}
force_inline void MixLine(const al::span<const float> InSamples, float *RESTRICT dst,
float &CurrentGain, const float TargetGain, const float delta, const size_t min_len,
const size_t aligned_len, size_t Counter)
{
float gain{CurrentGain};
const float step{(TargetGain-gain) * delta};
size_t pos{0};
if(!(std::abs(step) > std::numeric_limits<float>::epsilon()))
gain = TargetGain;
else
{
float step_count{0.0f};
/* Mix with applying gain steps in aligned multiples of 4. */
if(size_t todo{min_len >> 2})
{
const __m128 four4{_mm_set1_ps(4.0f)};
const __m128 step4{_mm_set1_ps(step)};
const __m128 gain4{_mm_set1_ps(gain)};
__m128 step_count4{_mm_setr_ps(0.0f, 1.0f, 2.0f, 3.0f)};
do {
const __m128 val4{_mm_load_ps(&InSamples[pos])};
__m128 dry4{_mm_load_ps(&dst[pos])};
/* dry += val * (gain + step*step_count) */
dry4 = vmadd(dry4, val4, vmadd(gain4, step4, step_count4));
_mm_store_ps(&dst[pos], dry4);
step_count4 = _mm_add_ps(step_count4, four4);
pos += 4;
} while(--todo);
/* NOTE: step_count4 now represents the next four counts after the
* last four mixed samples, so the lowest element represents the
* next step count to apply.
*/
step_count = _mm_cvtss_f32(step_count4);
}
/* Mix with applying left over gain steps that aren't aligned multiples of 4. */
for(size_t leftover{min_len&3};leftover;++pos,--leftover)
{
dst[pos] += InSamples[pos] * (gain + step*step_count);
step_count += 1.0f;
}
if(pos == Counter)
gain = TargetGain;
else
gain += step*step_count;
/* Mix until pos is aligned with 4 or the mix is done. */
for(size_t leftover{aligned_len&3};leftover;++pos,--leftover)
dst[pos] += InSamples[pos] * gain;
}
CurrentGain = gain;
if(!(std::abs(gain) > GainSilenceThreshold))
return;
if(size_t todo{(InSamples.size()-pos) >> 2})
{
const __m128 gain4{_mm_set1_ps(gain)};
do {
const __m128 val4{_mm_load_ps(&InSamples[pos])};
__m128 dry4{_mm_load_ps(&dst[pos])};
dry4 = _mm_add_ps(dry4, _mm_mul_ps(val4, gain4));
_mm_store_ps(&dst[pos], dry4);
pos += 4;
} while(--todo);
}
for(size_t leftover{(InSamples.size()-pos)&3};leftover;++pos,--leftover)
dst[pos] += InSamples[pos] * gain;
}
} // namespace
template<>
float *Resample_<BSincTag,SSETag>(const InterpState *state, float *RESTRICT src, uint frac,
uint increment, const al::span<float> dst)
void Resample_<CubicTag,SSETag>(const InterpState *state, const float *src, uint frac,
const uint increment, const al::span<float> dst)
{
const float *const filter{state->bsinc.filter};
const __m128 sf4{_mm_set1_ps(state->bsinc.sf)};
const size_t m{state->bsinc.m};
ASSUME(m > 0);
ASSUME(frac < MixerFracOne);
src -= state->bsinc.l;
for(float &out_sample : dst)
const auto filter = std::get<CubicState>(*state).filter;
src -= 1;
std::generate(dst.begin(), dst.end(), [&src,&frac,increment,filter]() -> float
{
const uint pi{frac >> CubicPhaseDiffBits}; ASSUME(pi < CubicPhaseCount);
const float pf{static_cast<float>(frac&CubicPhaseDiffMask) * (1.0f/CubicPhaseDiffOne)};
const __m128 pf4{_mm_set1_ps(pf)};
/* Apply the phase interpolated filter. */
/* f = fil + pf*phd */
const __m128 f4 = vmadd(_mm_load_ps(filter[pi].mCoeffs.data()), pf4,
_mm_load_ps(filter[pi].mDeltas.data()));
/* r = f*src */
__m128 r4{_mm_mul_ps(f4, _mm_loadu_ps(src))};
r4 = _mm_add_ps(r4, _mm_shuffle_ps(r4, r4, _MM_SHUFFLE(0, 1, 2, 3)));
r4 = _mm_add_ps(r4, _mm_movehl_ps(r4, r4));
const float output{_mm_cvtss_f32(r4)};
frac += increment;
src += frac>>MixerFracBits;
frac &= MixerFracMask;
return output;
});
}
template<>
void Resample_<BSincTag,SSETag>(const InterpState *state, const float *src, uint frac,
const uint increment, const al::span<float> dst)
{
const auto &bsinc = std::get<BsincState>(*state);
const float *const filter{bsinc.filter};
const __m128 sf4{_mm_set1_ps(bsinc.sf)};
const size_t m{bsinc.m};
ASSUME(m > 0);
ASSUME(frac < MixerFracOne);
src -= bsinc.l;
std::generate(dst.begin(), dst.end(), [&src,&frac,increment,filter,sf4,m]() -> float
{
// Calculate the phase index and factor.
const uint pi{frac >> FracPhaseBitDiff};
const float pf{static_cast<float>(frac & (FracPhaseDiffOne-1)) * (1.0f/FracPhaseDiffOne)};
const uint pi{frac >> BSincPhaseDiffBits};
const float pf{static_cast<float>(frac&BSincPhaseDiffMask) * (1.0f/BSincPhaseDiffOne)};
// Apply the scale and phase interpolated filter.
__m128 r4{_mm_setzero_ps()};
{
const __m128 pf4{_mm_set1_ps(pf)};
const float *RESTRICT fil{filter + m*pi*2};
const float *RESTRICT phd{fil + m};
const float *RESTRICT scd{fil + BSincPhaseCount*2*m};
const float *RESTRICT spd{scd + m};
const float *fil{filter + m*pi*2_uz};
const float *phd{fil + m};
const float *scd{fil + BSincPhaseCount*2_uz*m};
const float *spd{scd + m};
size_t td{m >> 2};
size_t j{0u};
do {
/* f = ((fil + sf*scd) + pf*(phd + sf*spd)) */
const __m128 f4 = MLA4(
MLA4(_mm_load_ps(&fil[j]), sf4, _mm_load_ps(&scd[j])),
pf4, MLA4(_mm_load_ps(&phd[j]), sf4, _mm_load_ps(&spd[j])));
const __m128 f4 = vmadd(
vmadd(_mm_load_ps(&fil[j]), sf4, _mm_load_ps(&scd[j])),
pf4, vmadd(_mm_load_ps(&phd[j]), sf4, _mm_load_ps(&spd[j])));
/* r += f*src */
r4 = MLA4(r4, f4, _mm_loadu_ps(&src[j]));
r4 = vmadd(r4, f4, _mm_loadu_ps(&src[j]));
j += 4;
} while(--td);
}
r4 = _mm_add_ps(r4, _mm_shuffle_ps(r4, r4, _MM_SHUFFLE(0, 1, 2, 3)));
r4 = _mm_add_ps(r4, _mm_movehl_ps(r4, r4));
out_sample = _mm_cvtss_f32(r4);
const float output{_mm_cvtss_f32(r4)};
frac += increment;
src += frac>>MixerFracBits;
frac &= MixerFracMask;
}
return dst.data();
return output;
});
}
template<>
float *Resample_<FastBSincTag,SSETag>(const InterpState *state, float *RESTRICT src, uint frac,
uint increment, const al::span<float> dst)
void Resample_<FastBSincTag,SSETag>(const InterpState *state, const float *src, uint frac,
const uint increment, const al::span<float> dst)
{
const float *const filter{state->bsinc.filter};
const size_t m{state->bsinc.m};
const auto &bsinc = std::get<BsincState>(*state);
const float *const filter{bsinc.filter};
const size_t m{bsinc.m};
ASSUME(m > 0);
ASSUME(frac < MixerFracOne);
src -= state->bsinc.l;
for(float &out_sample : dst)
src -= bsinc.l;
std::generate(dst.begin(), dst.end(), [&src,&frac,increment,filter,m]() -> float
{
// Calculate the phase index and factor.
const uint pi{frac >> FracPhaseBitDiff};
const float pf{static_cast<float>(frac & (FracPhaseDiffOne-1)) * (1.0f/FracPhaseDiffOne)};
const uint pi{frac >> BSincPhaseDiffBits};
const float pf{static_cast<float>(frac&BSincPhaseDiffMask) * (1.0f/BSincPhaseDiffOne)};
// Apply the phase interpolated filter.
__m128 r4{_mm_setzero_ps()};
{
const __m128 pf4{_mm_set1_ps(pf)};
const float *RESTRICT fil{filter + m*pi*2};
const float *RESTRICT phd{fil + m};
const float *fil{filter + m*pi*2_uz};
const float *phd{fil + m};
size_t td{m >> 2};
size_t j{0u};
do {
/* f = fil + pf*phd */
const __m128 f4 = MLA4(_mm_load_ps(&fil[j]), pf4, _mm_load_ps(&phd[j]));
const __m128 f4 = vmadd(_mm_load_ps(&fil[j]), pf4, _mm_load_ps(&phd[j]));
/* r += f*src */
r4 = MLA4(r4, f4, _mm_loadu_ps(&src[j]));
r4 = vmadd(r4, f4, _mm_loadu_ps(&src[j]));
j += 4;
} while(--td);
}
r4 = _mm_add_ps(r4, _mm_shuffle_ps(r4, r4, _MM_SHUFFLE(0, 1, 2, 3)));
r4 = _mm_add_ps(r4, _mm_movehl_ps(r4, r4));
out_sample = _mm_cvtss_f32(r4);
const float output{_mm_cvtss_f32(r4)};
frac += increment;
src += frac>>MixerFracBits;
frac &= MixerFracMask;
}
return dst.data();
return output;
});
}
@@ -183,7 +309,8 @@ void MixHrtfBlend_<SSETag>(const float *InSamples, float2 *AccumSamples, const u
template<>
void MixDirectHrtf_<SSETag>(const FloatBufferSpan LeftOut, const FloatBufferSpan RightOut,
const al::span<const FloatBufferLine> InSamples, float2 *AccumSamples,
float *TempBuf, HrtfChannelState *ChanState, const size_t IrSize, const size_t BufferSize)
const al::span<float,BufferLineSize> TempBuf, HrtfChannelState *ChanState, const size_t IrSize,
const size_t BufferSize)
{
MixDirectHrtfBase<ApplyCoeffs>(LeftOut, RightOut, InSamples, AccumSamples, TempBuf, ChanState,
IrSize, BufferSize);
@@ -195,78 +322,22 @@ void Mix_<SSETag>(const al::span<const float> InSamples, const al::span<FloatBuf
float *CurrentGains, const float *TargetGains, const size_t Counter, const size_t OutPos)
{
const float delta{(Counter > 0) ? 1.0f / static_cast<float>(Counter) : 0.0f};
const auto min_len = minz(Counter, InSamples.size());
const auto aligned_len = minz((min_len+3) & ~size_t{3}, InSamples.size()) - min_len;
const auto min_len = std::min(Counter, InSamples.size());
const auto aligned_len = std::min((min_len+3_uz) & ~3_uz, InSamples.size()) - min_len;
for(FloatBufferLine &output : OutBuffer)
{
float *RESTRICT dst{al::assume_aligned<16>(output.data()+OutPos)};
float gain{*CurrentGains};
const float step{(*TargetGains-gain) * delta};
size_t pos{0};
if(!(std::abs(step) > std::numeric_limits<float>::epsilon()))
gain = *TargetGains;
else
{
float step_count{0.0f};
/* Mix with applying gain steps in aligned multiples of 4. */
if(size_t todo{min_len >> 2})
{
const __m128 four4{_mm_set1_ps(4.0f)};
const __m128 step4{_mm_set1_ps(step)};
const __m128 gain4{_mm_set1_ps(gain)};
__m128 step_count4{_mm_setr_ps(0.0f, 1.0f, 2.0f, 3.0f)};
do {
const __m128 val4{_mm_load_ps(&InSamples[pos])};
__m128 dry4{_mm_load_ps(&dst[pos])};
/* dry += val * (gain + step*step_count) */
dry4 = MLA4(dry4, val4, MLA4(gain4, step4, step_count4));
_mm_store_ps(&dst[pos], dry4);
step_count4 = _mm_add_ps(step_count4, four4);
pos += 4;
} while(--todo);
/* NOTE: step_count4 now represents the next four counts after
* the last four mixed samples, so the lowest element
* represents the next step count to apply.
*/
step_count = _mm_cvtss_f32(step_count4);
}
/* Mix with applying left over gain steps that aren't aligned multiples of 4. */
for(size_t leftover{min_len&3};leftover;++pos,--leftover)
{
dst[pos] += InSamples[pos] * (gain + step*step_count);
step_count += 1.0f;
}
if(pos == Counter)
gain = *TargetGains;
else
gain += step*step_count;
/* Mix until pos is aligned with 4 or the mix is done. */
for(size_t leftover{aligned_len&3};leftover;++pos,--leftover)
dst[pos] += InSamples[pos] * gain;
}
*CurrentGains = gain;
++CurrentGains;
++TargetGains;
if(!(std::abs(gain) > GainSilenceThreshold))
continue;
if(size_t todo{(InSamples.size()-pos) >> 2})
{
const __m128 gain4{_mm_set1_ps(gain)};
do {
const __m128 val4{_mm_load_ps(&InSamples[pos])};
__m128 dry4{_mm_load_ps(&dst[pos])};
dry4 = _mm_add_ps(dry4, _mm_mul_ps(val4, gain4));
_mm_store_ps(&dst[pos], dry4);
pos += 4;
} while(--todo);
}
for(size_t leftover{(InSamples.size()-pos)&3};leftover;++pos,--leftover)
dst[pos] += InSamples[pos] * gain;
}
MixLine(InSamples, al::assume_aligned<16>(output.data()+OutPos), *CurrentGains++,
*TargetGains++, delta, min_len, aligned_len, Counter);
}
template<>
void Mix_<SSETag>(const al::span<const float> InSamples, float *OutBuffer, float &CurrentGain,
const float TargetGain, const size_t Counter)
{
const float delta{(Counter > 0) ? 1.0f / static_cast<float>(Counter) : 0.0f};
const auto min_len = std::min(Counter, InSamples.size());
const auto aligned_len = std::min((min_len+3_uz) & ~3_uz, InSamples.size()) - min_len;
MixLine(InSamples, al::assume_aligned<16>(OutBuffer), CurrentGain, TargetGain, delta, min_len,
aligned_len, Counter);
}
libs/openal-soft/core/mixer/mixer_sse2.cpp
+140 -20
View File
@@ -23,67 +23,187 @@
#include <xmmintrin.h>
#include <emmintrin.h>
#include <algorithm>
#include <array>
#include <cstddef>
#include <variant>
#include "alnumeric.h"
#include "alspan.h"
#include "core/cubic_defs.h"
#include "defs.h"
#include "opthelpers.h"
struct SSE2Tag;
struct LerpTag;
struct CubicTag;
#if defined(__GNUC__) && !defined(__clang__) && !defined(__SSE2__)
#pragma GCC target("sse2")
#endif
using uint = unsigned int;
namespace {
constexpr uint CubicPhaseDiffBits{MixerFracBits - CubicPhaseBits};
constexpr uint CubicPhaseDiffOne{1 << CubicPhaseDiffBits};
constexpr uint CubicPhaseDiffMask{CubicPhaseDiffOne - 1u};
force_inline __m128 vmadd(const __m128 x, const __m128 y, const __m128 z) noexcept
{ return _mm_add_ps(x, _mm_mul_ps(y, z)); }
} // namespace
template<>
float *Resample_<LerpTag,SSE2Tag>(const InterpState*, float *RESTRICT src, uint frac,
uint increment, const al::span<float> dst)
void Resample_<LerpTag,SSE2Tag>(const InterpState*, const float *src, uint frac,
const uint increment, const al::span<float> dst)
{
ASSUME(frac < MixerFracOne);
const __m128i increment4{_mm_set1_epi32(static_cast<int>(increment*4))};
const __m128 fracOne4{_mm_set1_ps(1.0f/MixerFracOne)};
const __m128i fracMask4{_mm_set1_epi32(MixerFracMask)};
alignas(16) uint pos_[4], frac_[4];
InitPosArrays(frac, increment, frac_, pos_);
alignas(16) std::array<uint,4> pos_, frac_;
InitPosArrays(frac, increment, al::span{frac_}, al::span{pos_});
__m128i frac4{_mm_setr_epi32(static_cast<int>(frac_[0]), static_cast<int>(frac_[1]),
static_cast<int>(frac_[2]), static_cast<int>(frac_[3]))};
__m128i pos4{_mm_setr_epi32(static_cast<int>(pos_[0]), static_cast<int>(pos_[1]),
static_cast<int>(pos_[2]), static_cast<int>(pos_[3]))};
auto dst_iter = dst.begin();
for(size_t todo{dst.size()>>2};todo;--todo)
auto vecout = al::span<__m128>{reinterpret_cast<__m128*>(dst.data()), dst.size()/4};
std::generate(vecout.begin(), vecout.end(), [=,&pos4,&frac4]() -> __m128
{
const int pos0{_mm_cvtsi128_si32(pos4)};
const int pos1{_mm_cvtsi128_si32(_mm_srli_si128(pos4, 4))};
const int pos2{_mm_cvtsi128_si32(_mm_srli_si128(pos4, 8))};
const int pos3{_mm_cvtsi128_si32(_mm_srli_si128(pos4, 12))};
const __m128 val1{_mm_setr_ps(src[pos0 ], src[pos1 ], src[pos2 ], src[pos3 ])};
const __m128 val2{_mm_setr_ps(src[pos0+1], src[pos1+1], src[pos2+1], src[pos3+1])};
const auto pos0 = static_cast<uint>(_mm_cvtsi128_si32(pos4));
const auto pos1 = static_cast<uint>(_mm_cvtsi128_si32(_mm_srli_si128(pos4, 4)));
const auto pos2 = static_cast<uint>(_mm_cvtsi128_si32(_mm_srli_si128(pos4, 8)));
const auto pos3 = static_cast<uint>(_mm_cvtsi128_si32(_mm_srli_si128(pos4, 12)));
const __m128 val1{_mm_setr_ps(src[pos0], src[pos1], src[pos2], src[pos3])};
const __m128 val2{_mm_setr_ps(src[pos0+1_uz], src[pos1+1_uz], src[pos2+1_uz], src[pos3+1_uz])};
/* val1 + (val2-val1)*mu */
const __m128 r0{_mm_sub_ps(val2, val1)};
const __m128 mu{_mm_mul_ps(_mm_cvtepi32_ps(frac4), fracOne4)};
const __m128 out{_mm_add_ps(val1, _mm_mul_ps(mu, r0))};
_mm_store_ps(dst_iter, out);
dst_iter += 4;
frac4 = _mm_add_epi32(frac4, increment4);
pos4 = _mm_add_epi32(pos4, _mm_srli_epi32(frac4, MixerFracBits));
frac4 = _mm_and_si128(frac4, fracMask4);
}
return out;
});
if(size_t todo{dst.size()&3})
{
src += static_cast<uint>(_mm_cvtsi128_si32(pos4));
frac = static_cast<uint>(_mm_cvtsi128_si32(frac4));
do {
*(dst_iter++) = lerpf(src[0], src[1], static_cast<float>(frac) * (1.0f/MixerFracOne));
std::generate(dst.end()-ptrdiff_t(todo), dst.end(), [&src,&frac,increment]()
{
const float out{lerpf(src[0], src[1], static_cast<float>(frac) * (1.0f/MixerFracOne))};
frac += increment;
src += frac>>MixerFracBits;
frac &= MixerFracMask;
} while(--todo);
return out;
});
}
}
template<>
void Resample_<CubicTag,SSE2Tag>(const InterpState *state, const float *src, uint frac,
const uint increment, const al::span<float> dst)
{
ASSUME(frac < MixerFracOne);
const auto filter = std::get<CubicState>(*state).filter;
const __m128i increment4{_mm_set1_epi32(static_cast<int>(increment*4))};
const __m128i fracMask4{_mm_set1_epi32(MixerFracMask)};
const __m128 fracDiffOne4{_mm_set1_ps(1.0f/CubicPhaseDiffOne)};
const __m128i fracDiffMask4{_mm_set1_epi32(CubicPhaseDiffMask)};
alignas(16) std::array<uint,4> pos_, frac_;
InitPosArrays(frac, increment, al::span{frac_}, al::span{pos_});
__m128i frac4{_mm_setr_epi32(static_cast<int>(frac_[0]), static_cast<int>(frac_[1]),
static_cast<int>(frac_[2]), static_cast<int>(frac_[3]))};
__m128i pos4{_mm_setr_epi32(static_cast<int>(pos_[0]), static_cast<int>(pos_[1]),
static_cast<int>(pos_[2]), static_cast<int>(pos_[3]))};
src -= 1;
auto vecout = al::span<__m128>{reinterpret_cast<__m128*>(dst.data()), dst.size()/4};
std::generate(vecout.begin(), vecout.end(), [=,&pos4,&frac4]() -> __m128
{
const auto pos0 = static_cast<uint>(_mm_cvtsi128_si32(pos4));
const auto pos1 = static_cast<uint>(_mm_cvtsi128_si32(_mm_srli_si128(pos4, 4)));
const auto pos2 = static_cast<uint>(_mm_cvtsi128_si32(_mm_srli_si128(pos4, 8)));
const auto pos3 = static_cast<uint>(_mm_cvtsi128_si32(_mm_srli_si128(pos4, 12)));
const __m128 val0{_mm_loadu_ps(src+pos0)};
const __m128 val1{_mm_loadu_ps(src+pos1)};
const __m128 val2{_mm_loadu_ps(src+pos2)};
const __m128 val3{_mm_loadu_ps(src+pos3)};
const __m128i pi4{_mm_srli_epi32(frac4, CubicPhaseDiffBits)};
const auto pi0 = static_cast<uint>(_mm_cvtsi128_si32(pi4));
const auto pi1 = static_cast<uint>(_mm_cvtsi128_si32(_mm_srli_si128(pi4, 4)));
const auto pi2 = static_cast<uint>(_mm_cvtsi128_si32(_mm_srli_si128(pi4, 8)));
const auto pi3 = static_cast<uint>(_mm_cvtsi128_si32(_mm_srli_si128(pi4, 12)));
ASSUME(pi0 < CubicPhaseCount); ASSUME(pi1 < CubicPhaseCount);
ASSUME(pi2 < CubicPhaseCount); ASSUME(pi3 < CubicPhaseCount);
const __m128 pf4{_mm_mul_ps(_mm_cvtepi32_ps(_mm_and_si128(frac4, fracDiffMask4)),
fracDiffOne4)};
__m128 r0{_mm_mul_ps(val0,
vmadd(_mm_load_ps(filter[pi0].mCoeffs.data()),
_mm_shuffle_ps(pf4, pf4, _MM_SHUFFLE(0, 0, 0, 0)),
_mm_load_ps(filter[pi0].mDeltas.data())))};
__m128 r1{_mm_mul_ps(val1,
vmadd(_mm_load_ps(filter[pi1].mCoeffs.data()),
_mm_shuffle_ps(pf4, pf4, _MM_SHUFFLE(1, 1, 1, 1)),
_mm_load_ps(filter[pi1].mDeltas.data())))};
__m128 r2{_mm_mul_ps(val2,
vmadd(_mm_load_ps(filter[pi2].mCoeffs.data()),
_mm_shuffle_ps(pf4, pf4, _MM_SHUFFLE(2, 2, 2, 2)),
_mm_load_ps(filter[pi2].mDeltas.data())))};
__m128 r3{_mm_mul_ps(val3,
vmadd(_mm_load_ps(filter[pi3].mCoeffs.data()),
_mm_shuffle_ps(pf4, pf4, _MM_SHUFFLE(3, 3, 3, 3)),
_mm_load_ps(filter[pi3].mDeltas.data())))};
_MM_TRANSPOSE4_PS(r0, r1, r2, r3);
r0 = _mm_add_ps(_mm_add_ps(r0, r1), _mm_add_ps(r2, r3));
frac4 = _mm_add_epi32(frac4, increment4);
pos4 = _mm_add_epi32(pos4, _mm_srli_epi32(frac4, MixerFracBits));
frac4 = _mm_and_si128(frac4, fracMask4);
return r0;
});
if(const size_t todo{dst.size()&3})
{
src += static_cast<uint>(_mm_cvtsi128_si32(pos4));
frac = static_cast<uint>(_mm_cvtsi128_si32(frac4));
std::generate(dst.end()-ptrdiff_t(todo), dst.end(), [&src,&frac,increment,filter]
{
const uint pi{frac >> CubicPhaseDiffBits}; ASSUME(pi < CubicPhaseCount);
const float pf{static_cast<float>(frac&CubicPhaseDiffMask) * (1.0f/CubicPhaseDiffOne)};
const __m128 pf4{_mm_set1_ps(pf)};
const __m128 f4 = vmadd(_mm_load_ps(filter[pi].mCoeffs.data()), pf4,
_mm_load_ps(filter[pi].mDeltas.data()));
__m128 r4{_mm_mul_ps(f4, _mm_loadu_ps(src))};
r4 = _mm_add_ps(r4, _mm_shuffle_ps(r4, r4, _MM_SHUFFLE(0, 1, 2, 3)));
r4 = _mm_add_ps(r4, _mm_movehl_ps(r4, r4));
const float output{_mm_cvtss_f32(r4)};
frac += increment;
src += frac>>MixerFracBits;
frac &= MixerFracMask;
return output;
});
}
return dst.data();
}
libs/openal-soft/core/mixer/mixer_sse41.cpp
+140 -20
View File
@@ -24,54 +24,76 @@
#include <emmintrin.h>
#include <smmintrin.h>
#include <algorithm>
#include <array>
#include <cstddef>
#include <variant>
#include "alnumeric.h"
#include "alspan.h"
#include "core/cubic_defs.h"
#include "defs.h"
#include "opthelpers.h"
struct SSE4Tag;
struct LerpTag;
struct CubicTag;
#if defined(__GNUC__) && !defined(__clang__) && !defined(__SSE4_1__)
#pragma GCC target("sse4.1")
#endif
using uint = unsigned int;
namespace {
constexpr uint CubicPhaseDiffBits{MixerFracBits - CubicPhaseBits};
constexpr uint CubicPhaseDiffOne{1 << CubicPhaseDiffBits};
constexpr uint CubicPhaseDiffMask{CubicPhaseDiffOne - 1u};
force_inline __m128 vmadd(const __m128 x, const __m128 y, const __m128 z) noexcept
{ return _mm_add_ps(x, _mm_mul_ps(y, z)); }
} // namespace
template<>
float *Resample_<LerpTag,SSE4Tag>(const InterpState*, float *RESTRICT src, uint frac,
uint increment, const al::span<float> dst)
void Resample_<LerpTag,SSE4Tag>(const InterpState*, const float *src, uint frac,
const uint increment, const al::span<float> dst)
{
ASSUME(frac < MixerFracOne);
const __m128i increment4{_mm_set1_epi32(static_cast<int>(increment*4))};
const __m128 fracOne4{_mm_set1_ps(1.0f/MixerFracOne)};
const __m128i fracMask4{_mm_set1_epi32(MixerFracMask)};
alignas(16) uint pos_[4], frac_[4];
InitPosArrays(frac, increment, frac_, pos_);
alignas(16) std::array<uint,4> pos_, frac_;
InitPosArrays(frac, increment, al::span{frac_}, al::span{pos_});
__m128i frac4{_mm_setr_epi32(static_cast<int>(frac_[0]), static_cast<int>(frac_[1]),
static_cast<int>(frac_[2]), static_cast<int>(frac_[3]))};
__m128i pos4{_mm_setr_epi32(static_cast<int>(pos_[0]), static_cast<int>(pos_[1]),
static_cast<int>(pos_[2]), static_cast<int>(pos_[3]))};
auto dst_iter = dst.begin();
for(size_t todo{dst.size()>>2};todo;--todo)
auto vecout = al::span<__m128>{reinterpret_cast<__m128*>(dst.data()), dst.size()/4};
std::generate(vecout.begin(), vecout.end(), [=,&pos4,&frac4]() -> __m128
{
const int pos0{_mm_extract_epi32(pos4, 0)};
const int pos1{_mm_extract_epi32(pos4, 1)};
const int pos2{_mm_extract_epi32(pos4, 2)};
const int pos3{_mm_extract_epi32(pos4, 3)};
const __m128 val1{_mm_setr_ps(src[pos0 ], src[pos1 ], src[pos2 ], src[pos3 ])};
const __m128 val2{_mm_setr_ps(src[pos0+1], src[pos1+1], src[pos2+1], src[pos3+1])};
const auto pos0 = static_cast<uint>(_mm_extract_epi32(pos4, 0));
const auto pos1 = static_cast<uint>(_mm_extract_epi32(pos4, 1));
const auto pos2 = static_cast<uint>(_mm_extract_epi32(pos4, 2));
const auto pos3 = static_cast<uint>(_mm_extract_epi32(pos4, 3));
const __m128 val1{_mm_setr_ps(src[pos0], src[pos1], src[pos2], src[pos3])};
const __m128 val2{_mm_setr_ps(src[pos0+1_uz], src[pos1+1_uz], src[pos2+1_uz], src[pos3+1_uz])};
/* val1 + (val2-val1)*mu */
const __m128 r0{_mm_sub_ps(val2, val1)};
const __m128 mu{_mm_mul_ps(_mm_cvtepi32_ps(frac4), fracOne4)};
const __m128 out{_mm_add_ps(val1, _mm_mul_ps(mu, r0))};
_mm_store_ps(dst_iter, out);
dst_iter += 4;
frac4 = _mm_add_epi32(frac4, increment4);
pos4 = _mm_add_epi32(pos4, _mm_srli_epi32(frac4, MixerFracBits));
frac4 = _mm_and_si128(frac4, fracMask4);
}
return out;
});
if(size_t todo{dst.size()&3})
{
@@ -82,13 +104,111 @@ float *Resample_<LerpTag,SSE4Tag>(const InterpState*, float *RESTRICT src, uint
src += static_cast<uint>(_mm_cvtsi128_si32(pos4));
frac = static_cast<uint>(_mm_cvtsi128_si32(frac4));
do {
*(dst_iter++) = lerpf(src[0], src[1], static_cast<float>(frac) * (1.0f/MixerFracOne));
std::generate(dst.end()-ptrdiff_t(todo), dst.end(), [&src,&frac,increment]
{
const float out{lerpf(src[0], src[1], static_cast<float>(frac) * (1.0f/MixerFracOne))};
frac += increment;
src += frac>>MixerFracBits;
frac &= MixerFracMask;
} while(--todo);
return out;
});
}
}
template<>
void Resample_<CubicTag,SSE4Tag>(const InterpState *state, const float *src, uint frac,
const uint increment, const al::span<float> dst)
{
ASSUME(frac < MixerFracOne);
const auto filter = std::get<CubicState>(*state).filter;
const __m128i increment4{_mm_set1_epi32(static_cast<int>(increment*4))};
const __m128i fracMask4{_mm_set1_epi32(MixerFracMask)};
const __m128 fracDiffOne4{_mm_set1_ps(1.0f/CubicPhaseDiffOne)};
const __m128i fracDiffMask4{_mm_set1_epi32(CubicPhaseDiffMask)};
alignas(16) std::array<uint,4> pos_, frac_;
InitPosArrays(frac, increment, al::span{frac_}, al::span{pos_});
__m128i frac4{_mm_setr_epi32(static_cast<int>(frac_[0]), static_cast<int>(frac_[1]),
static_cast<int>(frac_[2]), static_cast<int>(frac_[3]))};
__m128i pos4{_mm_setr_epi32(static_cast<int>(pos_[0]), static_cast<int>(pos_[1]),
static_cast<int>(pos_[2]), static_cast<int>(pos_[3]))};
src -= 1;
auto vecout = al::span<__m128>{reinterpret_cast<__m128*>(dst.data()), dst.size()/4};
std::generate(vecout.begin(), vecout.end(), [=,&pos4,&frac4]() -> __m128
{
const auto pos0 = static_cast<uint>(_mm_extract_epi32(pos4, 0));
const auto pos1 = static_cast<uint>(_mm_extract_epi32(pos4, 1));
const auto pos2 = static_cast<uint>(_mm_extract_epi32(pos4, 2));
const auto pos3 = static_cast<uint>(_mm_extract_epi32(pos4, 3));
const __m128 val0{_mm_loadu_ps(src+pos0)};
const __m128 val1{_mm_loadu_ps(src+pos1)};
const __m128 val2{_mm_loadu_ps(src+pos2)};
const __m128 val3{_mm_loadu_ps(src+pos3)};
const __m128i pi4{_mm_srli_epi32(frac4, CubicPhaseDiffBits)};
const auto pi0 = static_cast<uint>(_mm_extract_epi32(pi4, 0));
const auto pi1 = static_cast<uint>(_mm_extract_epi32(pi4, 1));
const auto pi2 = static_cast<uint>(_mm_extract_epi32(pi4, 2));
const auto pi3 = static_cast<uint>(_mm_extract_epi32(pi4, 3));
ASSUME(pi0 < CubicPhaseCount); ASSUME(pi1 < CubicPhaseCount);
ASSUME(pi2 < CubicPhaseCount); ASSUME(pi3 < CubicPhaseCount);
const __m128 pf4{_mm_mul_ps(_mm_cvtepi32_ps(_mm_and_si128(frac4, fracDiffMask4)),
fracDiffOne4)};
__m128 r0{_mm_mul_ps(val0,
vmadd(_mm_load_ps(filter[pi0].mCoeffs.data()),
_mm_shuffle_ps(pf4, pf4, _MM_SHUFFLE(0, 0, 0, 0)),
_mm_load_ps(filter[pi0].mDeltas.data())))};
__m128 r1{_mm_mul_ps(val1,
vmadd(_mm_load_ps(filter[pi1].mCoeffs.data()),
_mm_shuffle_ps(pf4, pf4, _MM_SHUFFLE(1, 1, 1, 1)),
_mm_load_ps(filter[pi1].mDeltas.data())))};
__m128 r2{_mm_mul_ps(val2,
vmadd(_mm_load_ps(filter[pi2].mCoeffs.data()),
_mm_shuffle_ps(pf4, pf4, _MM_SHUFFLE(2, 2, 2, 2)),
_mm_load_ps(filter[pi2].mDeltas.data())))};
__m128 r3{_mm_mul_ps(val3,
vmadd(_mm_load_ps(filter[pi3].mCoeffs.data()),
_mm_shuffle_ps(pf4, pf4, _MM_SHUFFLE(3, 3, 3, 3)),
_mm_load_ps(filter[pi3].mDeltas.data())))};
_MM_TRANSPOSE4_PS(r0, r1, r2, r3);
r0 = _mm_add_ps(_mm_add_ps(r0, r1), _mm_add_ps(r2, r3));
frac4 = _mm_add_epi32(frac4, increment4);
pos4 = _mm_add_epi32(pos4, _mm_srli_epi32(frac4, MixerFracBits));
frac4 = _mm_and_si128(frac4, fracMask4);
return r0;
});
if(const size_t todo{dst.size()&3})
{
src += static_cast<uint>(_mm_cvtsi128_si32(pos4));
frac = static_cast<uint>(_mm_cvtsi128_si32(frac4));
std::generate(dst.end()-ptrdiff_t(todo), dst.end(), [&src,&frac,increment,filter]
{
const uint pi{frac >> CubicPhaseDiffBits}; ASSUME(pi < CubicPhaseCount);
const float pf{static_cast<float>(frac&CubicPhaseDiffMask) * (1.0f/CubicPhaseDiffOne)};
const __m128 pf4{_mm_set1_ps(pf)};
const __m128 f4 = vmadd(_mm_load_ps(filter[pi].mCoeffs.data()), pf4,
_mm_load_ps(filter[pi].mDeltas.data()));
__m128 r4{_mm_mul_ps(f4, _mm_loadu_ps(src))};
r4 = _mm_add_ps(r4, _mm_shuffle_ps(r4, r4, _MM_SHUFFLE(0, 1, 2, 3)));
r4 = _mm_add_ps(r4, _mm_movehl_ps(r4, r4));
const float output{_mm_cvtss_f32(r4)};
frac += increment;
src += frac>>MixerFracBits;
frac &= MixerFracMask;
return output;
});
}
return dst.data();
}