文本是在项目实施过程中对音频频域可视化进行测试时的一些记录
后附麦克风API介绍, 资料与版权声明
基本步骤:

最终效果
为了提取音频中的频域信号, 我们首先对音频执行傅里叶变换
我在stackoverflow上找到了一个简单的fft(快速傅里叶变换)实现
https://stackoverflow.com/questions/170394/an-implementation-of-the-fast-fourier-transform-fft-in-c-sharp
相比原文, 修改了函数名使其符合C#命名规范. 注意文件头的MIT版权声明
代码块
using System;
/**
* Performs an in-place complex FFT.
*
* Released under the MIT License
*
* Copyright (c) 2010 Gerald T. Beauregard
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
public class FFT2
{
// Element for linked list in which we store the
// input/output data. We use a linked list because
// for sequential access it's faster than array index.
class FFTElement
{
public double re = 0.0; // Real component
public double im = 0.0; // Imaginary component
public FFTElement next; // Next element in linked list
public uint revTgt; // Target position post bit-reversal
}
private uint m_logN = 0; // log2 of FFT size
private uint m_N = 0; // FFT size
private FFTElement[] m_X; // Vector of linked list elements
/**
*
*/
public FFT2()
{
}
/**
* Initialize class to perform FFT of specified size.
*
* @param logN Log2 of FFT length. e.g. for 512 pt FFT, logN = 9.
*/
public void Init(
uint logN)
{
m_logN = logN;
m_N = (uint)(1 << (int)m_logN);
// Allocate elements for linked list of complex numbers.
m_X = new FFTElement[m_N];
for (uint k = 0; k < m_N; k++)
m_X[k] = new FFTElement();
// Set up "next" pointers.
for (uint k = 0; k < m_N - 1; k++)
m_X[k].next = m_X[k + 1];
// Specify target for bit reversal re-ordering.
for (uint k = 0; k < m_N; k++)
m_X[k].revTgt = BitReverse(k, logN);
}
/**
* Performs in-place complex FFT.
*
* @param xRe Real part of input/output
* @param xIm Imaginary part of input/output
* @param inverse If true, do an inverse FFT
*/
public void Run(
double[] xRe,
double[] xIm,
bool inverse = false)
{
uint numFlies = m_N >> 1; // Number of butterflies per sub-FFT
uint span = m_N >> 1; // Width of the butterfly
uint spacing = m_N; // Distance between start of sub-FFTs
uint wIndexStep = 1; // Increment for twiddle table index
// Copy data into linked complex number objects
// If it's an IFFT, we divide by N while we're at it
FFTElement x = m_X[0];
uint k = 0;
double scale = inverse ? 1.0 / m_N : 1.0;
while (x != null)
{
x.re = scale * xRe[k];
x.im = scale * xIm[k];
x = x.next;
k++;
}
// For each stage of the FFT
for (uint stage = 0; stage < m_logN; stage++)
{
// Compute a multiplier factor for the "twiddle factors".
// The twiddle factors are complex unit vectors spaced at
// regular angular intervals. The angle by which the twiddle
// factor advances depends on the FFT stage. In many FFT
// implementations the twiddle factors are cached, but because
// array lookup is relatively slow in C#, it's just
// as fast to compute them on the fly.
double wAngleInc = wIndexStep * 2.0 * Math.PI / m_N;
if (inverse == false)
wAngleInc *= -1;
double wMulRe = Math.Cos(wAngleInc);
double wMulIm = Math.Sin(wAngleInc);
for (uint start = 0; start < m_N; start += spacing)
{
FFTElement xTop = m_X[start];
FFTElement xBot = m_X[start + span];
double wRe = 1.0;
double wIm = 0.0;
// For each butterfly in this stage
for (uint flyCount = 0; flyCount < numFlies; ++flyCount)
{
// Get the top & bottom values
double xTopRe = xTop.re;
double xTopIm = xTop.im;
double xBotRe = xBot.re;
double xBotIm = xBot.im;
// Top branch of butterfly has addition
xTop.re = xTopRe + xBotRe;
xTop.im = xTopIm + xBotIm;
// Bottom branch of butterly has subtraction,
// followed by multiplication by twiddle factor
xBotRe = xTopRe - xBotRe;
xBotIm = xTopIm - xBotIm;
xBot.re = xBotRe * wRe - xBotIm * wIm;
xBot.im = xBotRe * wIm + xBotIm * wRe;
// Advance butterfly to next top & bottom positions
xTop = xTop.next;
xBot = xBot.next;
// Update the twiddle factor, via complex multiply
// by unit vector with the appropriate angle
// (wRe + j wIm) = (wRe + j wIm) x (wMulRe + j wMulIm)
double tRe = wRe;
wRe = wRe * wMulRe - wIm * wMulIm;
wIm = tRe * wMulIm + wIm * wMulRe;
}
}
numFlies >>= 1; // Divide by 2 by right shift
span >>= 1;
spacing >>= 1;
wIndexStep <<= 1; // Multiply by 2 by left shift
}
// The algorithm leaves the result in a scrambled order.
// Unscramble while copying values from the complex
// linked list elements back to the input/output vectors.
x = m_X[0];
while (x != null)
{
uint target = x.revTgt;
xRe[target] = x.re;
xIm[target] = x.im;
x = x.next;
}
}
/**
* Do bit reversal of specified number of places of an int
* For example, 1101 bit-reversed is 1011
*
* @param x Number to be bit-reverse.
* @param numBits Number of bits in the number.
*/
private uint BitReverse(
uint x,
uint numBits)
{
uint y = 0;
for (uint i = 0; i < numBits; i++)
{
y <<= 1;
y |= x & 0x0001;
x >>= 1;
}
return y;
}
}
标准的fft实现的结果是一个复数数组
我们需要对接口进行一些包装方便稍后处理
-> 输入AudioClip中提取的归一化flaot数据
-> 输出以[(频率, 幅值)]形式表示的频谱数据
代码块
// 包装fft函数
// 输入[-1,1]归一化的pcm数据
// 返回频谱数据, 频谱数据以<频率, 幅值>的形式给出
public static void FFT(System.Span<float> audioData, int sampleRate, ref List<Vector2> spectrum)
{
// 确定FFT大的大小, 限制为2的幂
int logN = (int)Mathf.Log(audioData.Length, 2);
if (Mathf.Pow(2, logN) < audioData.Length)
logN++;
// 2的logN次方
uint fftSize = (uint)(1 << logN);
var fft = new FFT2();
fft.Init((uint)logN);
// 将输入的归一化音频数据填充到复数数组
// 虚部总是为0
double[] xRe = new double[fftSize];
double[] xIm = new double[fftSize];
for (int i = 0; i < audioData.Length && i < fftSize; i++)
{
xRe[i] = audioData[i];
xIm[i] = 0.0;
}
// 如果音频数据长度小于FFT大小,剩余部分填充0
for (int i = audioData.Length; i < fftSize; i++)
{
xRe[i] = 0.0;
xIm[i] = 0.0;
}
// 执行FFT
fft.Run(xRe, xIm, inverse: false);
// 结果将是一个以0为中心的对称数据
// 只取前半部分
int spectrumSize = (int)(fftSize / 2);
float frequencyStep = sampleRate / (2.0f * spectrumSize);
// 清除缓冲区准备写入
spectrum.Clear();
for (int i = 0; i < spectrumSize; i++)
{
// 计算复数的幅值:sqrt(re^2 + im^2)
float amplitude = Mathf.Sqrt((float)(xRe[i] * xRe[i] + xIm[i] * xIm[i]));
// 对应频率
float frequency = i * frequencyStep;
// 需要的话, 转换为分贝值(dB)
// amplitude = 20 * Mathf.Log10(amplitude + Mathf.Epsilon);
spectrum.Add(new Vector2(frequency, amplitude));
}
}
由于我们期望的视觉效果是一组柱状图, 而不是Matplotlib中精确的折线图
根据不同的参数, fft返回的数据将包含数万个采样数据, 而我们只有不到50个柱形
所以我们需要对该函数返回的fft数据再进行一次均值降采样, 我们需要在外部循环中计算出所需的centreFreq和range并输入此函数进行采样
代码块
// 对采样数据执行均值重采样
// 对spectrum提供的频谱数据进行二次采样, 提取以centreFreq为中心, range为范围内的采样值的均值
// 包含一个高通滤波器选项, 用于滤除低于highPass的频率成分
public static float MeanResampling(List<Vector2> spectrum, float centreFreq, float range, float? highPass = null)
{
Debug.Assert(spectrum.Count >= 2);
if (spectrum.Count == 0) return 0;
// 将采样范围映射到索引范围
float startFreq = centreFreq - range / 2;
float endFreq = centreFreq + range / 2;
int startIndex = spectrum.FindIndex(t => t.x >= startFreq);
int end = spectrum.FindIndex(t => t.x > endFreq);
// 处理边界情况
if (startIndex < 0)
return 0;
if (end < 0)
end = spectrum.Count;
if (startIndex == end)
return spectrum[startIndex].y;
// 简单取区间均值, 而不做平滑窗
float sum = 0;
int count = 0;
for (int i = startIndex; i < end; i++)
{
if (highPass.HasValue && spectrum[i].x < highPass)
continue;
sum += spectrum[i].y;
count++;
}
return sum / count;
}
在场景中创建一个Horiznontal和若干Image, 调整好参数, 并将所有Image填充到m_bars
同时为了创建动画效果或是进行插值, 额外创建一个m_barValue数组来储存bar的目标值
代码块
[SerializeField] private Image[] m_bars;
List<Vector2> m_spectrum = new List<Vector2>();
private float[] m_barValue = null;
流程测试代码, 使用了UniTask, 应该可以很方便的转写为协程或是update风格的流程代码
代码块
private async void Awake()
{
m_barValue = new float[m_bars.Length];
// 等待按下Space
await UniTask.WaitUntil(() => Input.GetKeyDown(KeyCode.Space));
// 检查设备是否可用
var devices = Microphone.devices;
if (devices.Length == 0)
throw new System.InvalidOperationException("Not found Micorphone device");
string deviceName = devices[0];
// 开始录音
const int MaxRecordingTime = 20;
int sampleRate = 24000;
var clip = Microphone.Start(deviceName, false, MaxRecordingTime, sampleRate);
Debug.Log($"start recording, sample rate {sampleRate}");
// 初始化小数组, 该数组负责储存录音中最后SegDuration秒的采样数据
const float SegDuration = 0.2f;
int segCount = Mathf.FloorToInt(SegDuration * sampleRate);
var segData = new float[segCount];
// 等待一段时间, 让缓冲区中有数据
await UniTask.WaitForSeconds(SegDuration);
while (true)
{
// 每步执行
// 一帧为一步
await UniTask.Yield();
// 录音结束后退出
if (!Microphone.IsRecording(deviceName))
break;
// 获取最后一段时间内的数据
int offset = Microphone.GetPosition(deviceName) - segData.Length;
// Debug.Log($"{offset}, {Microphone.GetPosition(deviceName)}, {segData.Length}");
// 在读取数据无效或是数据不足时返回等待
if (offset < 0) continue;
if (!clip.GetData(segData, offset)) continue;
// 创建处理器并处理音频
// 生成频谱数据与频率表
// 频率表与频谱数据一一对应, 频率表的值代表该项的频率, 频谱数据的值代表该频率的强度
FFT(segData, sampleRate, ref m_spectrum);
// 柱状图的显示范围
// 显示数据中n-3000HZ的频率成分
// 简单的忽略信号中的直流偏置, 采样信号总是带有直流偏置, 在频域上显示为强烈的0频率信号
int displayFreqMin = 50;
int displayFreqMax = 3000;
// 由于稍后将执行均值采样, 幅值会衰退的很厉害, 所以也适当的降低max防止归一化的结果太小
const float MaxAmplitudeFactor = 0.3f;
// 计算目标频域范围中的最大幅值用于归一化
var it = m_spectrum
.SkipWhile(t => t.x < displayFreqMin)
.TakeWhile(t => t.x <= displayFreqMax);
if (!it.Any()) continue;
// 设置一个最小值防止max过小, 消除底噪的影响
float max = Mathf.Max(it.Max(t => t.y) * MaxAmplitudeFactor, 3);
// 将值归一化并储存到barValue
// 稍后在Update中插值同步
int sampleCount = m_bars.Length;
float freqStep = displayFreqMax / sampleCount;
for (int i = 0; i < sampleCount; i++)
{
float t = (float)i / sampleCount;
float freq = Mathf.Lerp(displayFreqMin, displayFreqMax, t);
float a = MeanResampling(m_spectrum, freq, freqStep, displayFreqMin);
float nor = Mathf.Clamp01(a / max);
// 处理浮动的极小值
const float Epsilon = 0.08f;
if (nor < Epsilon)
nor = 0;
m_barValue[i] = nor;
}
}
// 在接收后将数据清零
System.Array.Fill(m_barValue, 0);
Debug.Log("Done");
}
另外在Update中同步视觉效果
代码块
// 额外添加一些颜色变换
[Space, SerializeField] private Color m_beginColor;
[SerializeField] private Color m_endColor;
private void Update()
{
const float BaseValue = 0.1f;
int sampleCount = m_bars.Length;
for (int i = 0; i < sampleCount; i++)
{
// 使用transform.locslScale实现效果并做插值
var t = m_bars[i].transform.localScale;
t.y = Mathf.MoveTowards(t.y, m_barValue[i] + BaseValue, 5.0f * Time.deltaTime);
m_bars[i].transform.localScale = t;
m_bars[i].color = Color.Lerp(m_beginColor, m_endColor, m_barValue[i]);
}
}
麦克风基础API
代码块
// 列出所有可以使用的麦克风名称
string[] devices = Microphone.devices;
// 列出目标设备的采样频率范围
string m = Microphone.devices[0];
Microphone.GetDeviceCaps(m, out int minFreq, out int maxFreq);
Debug.Log($"Mic device: {m}, {minFreq} - {maxFreq}");
// 启动麦克风录音
// 指定采样率为sampleRate, 最长录音时间为maxDuration, 在maxDuration之后是否继续录音, 新数据将回绕覆盖缓冲区
// 该函数将创建并返回一个长度为maxDuration的AudioClip, 样本数据为归一化的float 范围[-1,1], 样本数量为maxDuration * sampleRate
int sampleRate = 24000;
int maxDuration = 20;
var clip = Microphone.Start(deviceName, false, maxDuration, sampleRate);
// 检查麦克风是否正在录音
bool recording = Microphone.IsRecording(deviceName);
// 获取麦克风的录音位置
int sampleIndex = Microphone.GetPosition(deviceName);
// 强制麦克风停止录音
Microphone.End(deviceName);
其他:
声波是一种在弹性介质中传播的机械波。
与无线电波一样,声波也具有波长、频率、振幅、波速等波的基本性质,遵循波的反射、折射、干涉等规律。
为了将声波这种连续变化的模拟信号转换为计算机等设备可处理的离散数字信号,我们需要按一定时间间隔对模拟信号的幅值进行采样,并对采样得到的幅值进行量化。
根据奈奎斯特定理(Nyquist Theorem),为避免采样后出现 “混叠失真”,采样频率至少应大于被采样信号最高频率的两倍,此时才能通过后续处理完整还原原始信号。
由于人类可听声音的频率范围普遍为 20Hz~20kHz(即最高频率为 20kHz),理论上采样频率需至少大于 40kHz。而普通麦克风及音频设备大多采用 44.1kHz 的采样频率,除满足奈奎斯特条件外,还源于早期 CD 音频标准的制定。
dsp数字信号处理教程提供了更多基础知识
https://pysdr.org/zh/content-zh/intro.html
https://zh.wikipedia.org/wiki/%E9%87%87%E6%A0%B7%E5%AE%9A%E7%90%86
fft的wiki
https://en.wikipedia.org/wiki/Fast_Fourier_transform
声明:
本文中未特别标注来源的代码均为本人原创。任何个人或组织使用这些代码(包括修改、二次分发等)时,需在使用场景中明确提及原作者,并附带本文的原始链接,以示来源。
文章中涉及的非本人原创内容(包括但不限于引用的第三方代码、库文件等),其知识产权及使用授权均遵循原作者或原版权方的相关协议。
免责声明:本文系网络转载或改编,未找到原创作者,版权归原作者所有。如涉及版权,请联系删