<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library\av_processing;
/**
* SpeechDecoder turns a recording made the way built for speech into
* the samples that are played.
*
* A browser records a voice message with the Opus audio codec, which
* holds two ways of compressing sound. This class reads the one built
* for speech, which is what a browser uses for a voice. For each stretch
* of twenty milliseconds it reads the opening, the shape of the mouth
* that made the sound, the pitch of the voice, and the pulses, then puts
* those back together into samples.
*
* The sound of one stretch leans on the stretch before it in two ways,
* so this class keeps that state as it goes: the values the shape filter
* holds, and the samples already played, which a voiced stretch reaches
* back into for its pitch.
*
* A caller hands it the packets of a recording and gets back whole
* numbers at sixteen thousand samples a second.
*
* @author Chris Pollett
*/
class SpeechDecoder
{
/**
* SAMPLE_RATE is how many samples a second the sound comes back at.
* A browser records speech at this rate, and a caller that wants
* another rate raises the samples afterward.
*/
const SAMPLE_RATE = 16000;
/**
* SAMPLES_IN_STRETCH is how many samples one stretch of twenty
* milliseconds covers at that rate.
*/
const SAMPLES_IN_STRETCH = 320;
/**
* QUARTERS_IN_STRETCH is how many quarters a stretch is cut into,
* each with its own loudness, pitch and filter.
*/
const QUARTERS_IN_STRETCH = 4;
/**
* STRETCHES_KEPT is how many stretches of played samples are kept.
* A voiced stretch reaches back into them for its pitch, and running
* them backward through the filter needs samples before the oldest
* one it reaches, so more is kept than is reached into.
*/
const STRETCHES_KEPT = 2;
/**
* BLEND_QUARTER is the quarter at which a stretch stops using the
* shape blended with the stretch before and starts using its own.
*/
const BLEND_QUARTER = 2;
/**
* HEADER_MARK is the eight letters at the front of the packet that
* carries a recording's settings rather than any sound.
*/
const HEADER_MARK = "OpusHead";
/**
* HEADER_RATE is the rate the header counts its samples at, which
* is forty-eight thousand a second whatever rate the sound itself
* was written at.
*/
const HEADER_RATE = 48000;
/**
* SMOOTHING_REACH is how many samples either side of a place are
* counted when a sample is made between two of them. Four either
* side is enough that a listener hears none of the sound that does
* not belong.
*/
const SMOOTHING_REACH = 4;
/**
* SMALLEST_GAP is how close two places have to be before they count
* as the same place, which keeps the curve from being asked to
* divide by nothing.
*/
const SMALLEST_GAP = 0.000000001;
/**
* $filter_state stores the values the shape filter is holding, which
* carry from one quarter to the next and across stretches.
* @var array
*/
public $filter_state = [];
/**
* $last_scale stores the multiplier the quarter before this one was
* played at, so the held values can be moved when the scale changes.
* @var int
*/
public $last_scale = 0;
/**
* $last_frequencies stores the sixteen frequencies of the stretch
* before this one, which the first half of this stretch blends with.
* @var array
*/
public $last_frequencies = [];
/**
* $last_played stores the samples of the stretch before this one. A
* voiced stretch reaches into them for its pitch.
* @var array
*/
public $last_played = [];
/**
* $stretches_read stores how many stretches this decoder has read,
* which a caller uses to say where in a recording it has got to.
* @var int
*/
public $stretches_read = 0;
/**
* $spare_copies_seen stores how many stretches carried a spare copy
* of themselves for a listener who lost the packet before. The
* copies are read past rather than played, since a recording being
* made over loses no packets.
* @var int
*/
public $spare_copies_seen = 0;
/**
* __construct sets the decoder up with nothing behind it, as it
* stands before the first stretch of a recording.
*/
public function __construct()
{
$this->filter_state = [];
$this->last_scale = 0;
$this->last_frequencies = [];
$this->last_played = array_fill(0,
self::SAMPLES_IN_STRETCH * self::STRETCHES_KEPT, 0);
$this->stretches_read = 0;
$this->spare_copies_seen = 0;
}
/**
* samplesOfRecording gives the samples of a whole recording, in
* order. A caller that has a file of sound compressed with the Opus
* audio codec uses this; it walks the file's packets, skips any that
* were compressed the way built for music, and joins what is left.
*
* @param string $path the file holding the recording
* @return array whole numbers, one for each sample, at sixteen
* thousand samples a second
*/
public static function samplesOfRecording($path)
{
$pieces = [];
$dropping = 0;
$reader = OggDemuxer::fromName($path);
foreach ($reader->packets() as $packet) {
if (substr($packet->data, 0, 8) === self::HEADER_MARK) {
$dropping = self::samplesToDrop($packet->data);
continue;
}
$pieces[] = $packet->data;
}
return self::samplesOfPackets($pieces, $dropping);
}
/**
* samplesOfPackets gives the samples of a run of packets already
* taken out of a file. A caller that has read the packets itself
* uses this, which is how a recording held in a WebM file is
* decoded: the file's own reader hands over the packets and this
* class never opens the file.
*
* @param array $pieces the compressed packets, in playing order
* @param int $dropping how many samples to drop from the front,
* which a recording's settings say
* @return array whole numbers, one for each sample, at sixteen
* thousand samples a second
*/
public static function samplesOfPackets($pieces, $dropping = 0)
{
$decoder = new self();
$samples = [];
foreach ($pieces as $piece) {
if (substr($piece, 0, 8) === self::HEADER_MARK) {
$dropping = self::samplesToDrop($piece);
continue;
}
try {
$sound = OpusPacket::fromString($piece);
} catch (\Exception $trouble) {
continue;
}
if ($sound->method == OpusPacket::MUSIC_METHOD) {
continue;
}
if ($sound->spectrum !== OpusPacket::WIDE_SPECTRUM) {
/* Only the widest of the three speech bands is read so
far. The narrower two hold ten filter terms rather
than sixteen and count their samples at eight or
twelve thousand a second, so reading them as this one
gives sound that is not speech at all. */
throw new \RuntimeException("this recording keeps only "
. "the " . $sound->spectrum . " part of the sound, "
. "which is not read yet");
}
foreach ($sound->stretches as $stretch) {
foreach ($decoder->samplesOfStretch($stretch) as $one) {
$samples[] = $one;
}
}
$decoder->stretches_read += count($sound->stretches);
}
if ($dropping > 0) {
$samples = array_slice($samples, $dropping);
}
return $samples;
}
/**
* samplesToDrop says how many samples a recording's settings ask to
* be dropped from the front of the sound. Those samples cover the
* time the writer's own filters took to settle, and a listener is
* meant never to hear them.
*
* @param string $settings the packet holding a recording's settings
* @return int how many samples to drop, counted at the rate the
* sound is decoded at
*/
public static function samplesToDrop($settings)
{
if (strlen($settings) < 12) {
return 0;
}
$asked = unpack("v", substr($settings, 10, 2));
return intdiv($asked[1],
intdiv(self::HEADER_RATE, self::SAMPLE_RATE));
}
/**
* samplesOfStretch gives the samples of one stretch of sound. It
* reads the stretch's opening, shape, pitch and pulses, turns the
* pulses into a rough sound, adds back the pitch where the stretch
* holds a voice, and passes the whole through the filter the shape
* describes.
*
* The decoder keeps what one stretch leaves for the next, so a
* caller reads the stretches of a recording in order and does not
* reuse a decoder across two recordings.
*
* @param string $stretch the compressed bytes of one stretch
* @return array whole numbers, one for each sample of the stretch
*/
public function samplesOfStretch($stretch)
{
$reader = new RangeDecoder($stretch);
$marks = SpeechFrameHeader::readMarks($reader, 1);
if (!empty($marks->spare_copies[0])) {
/* A spare copy of this stretch is written before the
stretch itself, for a listener who lost the packet
before. A recording being made over loses nothing, so
the copy is read past and thrown away; reading past it
is what puts the reader at the stretch's own bytes. */
$this->spare_copies_seen++;
self::readOneStretch($reader, true);
}
list($head, $shape, $pitch, $pulses) =
self::readOneStretch($reader, $marks->has_speech[0]);
$rough = SpeechSamples::roughSound($pulses->samples,
$pulses->seed, $head->kind, $head->pulse_writing);
$whole_terms = SpeechFilter::termsFor($shape->frequencies);
$half_terms = SpeechFilter::termsFor(SpeechFilter::blendedWith(
$shape->frequencies, $this->last_frequencies,
$shape->blend));
$this->last_frequencies = $shape->frequencies;
$found = SpeechLoudness::scalesFor($head->loudness_indices);
if (count($this->filter_state) != count($whole_terms)) {
$this->filter_state = array_fill(0, count($whole_terms), 0);
}
$carried = [];
if ($pitch !== null) {
$backward = SpeechSamples::soundRunBackward(
$this->last_played, $half_terms);
$carried = SpeechSamples::atFilterScale($backward,
$found["scales"][0], $pitch->lean);
}
$played = [];
$each = intdiv(self::SAMPLES_IN_STRETCH,
self::QUARTERS_IN_STRETCH);
for ($quarter = 0; $quarter < self::QUARTERS_IN_STRETCH;
$quarter++) {
$piece = array_slice($rough, $quarter * $each, $each);
$terms = ($quarter < self::QUARTERS_IN_STRETCH / 2) ?
$half_terms : $whole_terms;
if ($pitch !== null && $quarter == self::BLEND_QUARTER &&
$shape->blend < SpeechFilter::WHOLE_BLEND) {
/* Where the first half of a stretch was played with a
blended shape, the second half starts again from the
stretch's own shape, so the sound it reaches into is
built again with that shape. */
$so_far = $this->last_played;
foreach ($played as $one) {
$so_far[] = $one;
}
$backward = SpeechSamples::soundRunBackward($so_far,
$whole_terms);
$carried = SpeechSamples::atFilterScale($backward,
$found["scales"][$quarter]);
} else if ($pitch !== null) {
/* The sound a stretch reaches into was scaled by the
loudness of the quarter that wrote it, so it moves
with the loudness the same way the filter's own held
values do. */
if ($quarter > 0) {
$carried = SpeechSamples::stateAtNewScale($carried,
$found["scales"][$quarter - 1],
$found["scales"][$quarter]);
}
}
if ($pitch !== null) {
$added = SpeechSamples::withPitchAdded($piece, $carried,
$pitch->quarter_lags[$quarter],
$pitch->quarter_filters[$quarter]);
$piece = $added["sound"];
$carried = $added["carried"];
}
$out = SpeechSamples::throughShapeFilter($piece, $terms,
$found["scales"][$quarter], $this->filter_state,
$this->last_scale);
$this->last_scale = $found["scales"][$quarter];
$this->filter_state = $out["state"];
foreach ($out["samples"] as $one) {
$played[] = $one;
}
}
$keeping = $this->last_played;
foreach ($played as $one) {
$keeping[] = $one;
}
$this->last_played = array_slice($keeping,
-self::SAMPLES_IN_STRETCH * self::STRETCHES_KEPT);
return $played;
}
/**
* readOneStretch reads the four things a stretch of speech carries:
* what its opening says, the shape it was spoken with, the pitch of
* a voiced one, and its pulses. A caller uses it both for a stretch
* it means to play and for a spare copy it means to throw away,
* since reading past a spare copy is the only way to reach the
* stretch behind it.
*
* @param object $reader the range decoder reading the stretch
* @param bool $has_speech whether the stretch was marked as holding
* speech
* @return array the opening, the shape, the pitch or null, and the
* pulses, in that order
*/
public static function readOneStretch($reader, $has_speech)
{
$head = SpeechFrameHeader::readSound($reader, $has_speech);
$shape = SpeechShape::read($reader, $head->kind);
$pitch = null;
if ($head->kind === SpeechTables::VOICED_KIND) {
$pitch = SpeechPitch::read($reader, -1, true);
}
$pulses = SpeechPulses::read($reader, $head->kind,
$head->pulse_writing, self::SAMPLES_IN_STRETCH);
return [$head, $shape, $pitch, $pulses];
}
/**
* raisedToRate gives the same sound at more samples a second, by
* drawing a straight line between each pair of samples and reading
* points along it. A recording is decoded at sixteen thousand
* samples a second, and a file that holds it usually says
* forty-eight thousand, so the sound has to be raised before it is
* written.
*
* The straight line is not what a careful resampler does, and a
* listener with a good ear may hear the difference on music. On
* speech at these rates it is not heard, and it costs one multiply
* a sample rather than a filter over many.
*
* @param array $samples the samples as decoded
* @param int $from how many samples a second they were decoded at
* @param int $to how many samples a second are wanted
* @return array the samples at the wanted rate
*/
public static function raisedToRate($samples, $from, $to)
{
if ($from == $to || $from < 1 || count($samples) < 2) {
return $samples;
}
$out = [];
$wanted = (int)floor(count($samples) * $to / $from);
$held = count($samples);
/* A straight line drawn between two samples leaves copies of
the sound above where the ear expects it, which a listener
hears as a hiss. Weighing several samples either side by a
curve that falls away leaves those out. */
$reach = self::SMOOTHING_REACH;
for ($at = 0; $at < $wanted; $at++) {
$where = $at * $from / $to;
$middle = (int)floor($where);
$sum = 0.0;
$weights = 0.0;
for ($step = -$reach; $step <= $reach; $step++) {
$which = $middle + $step;
if ($which < 0 || $which >= $held) {
continue;
}
$weight = self::smoothingWeight($where - $which, $reach);
$sum += $samples[$which] * $weight;
$weights += $weight;
}
$out[] = ($weights > 0) ?
max(-32768, min(32767, (int)round($sum / $weights))) : 0;
}
return $out;
}
/**
* smoothingWeight says how much one sample counts toward a sample
* being made between two of them. The weight follows the curve a
* run of samples adds back up to, narrowed by a window so that only
* a few samples either side are needed.
*
* @param float $away how far the sample sits from the place being
* filled, counted in samples
* @param int $reach how many samples either side are counted
* @return float how much this sample counts, from nothing upward
*/
public static function smoothingWeight($away, $reach)
{
if (abs($away) < self::SMALLEST_GAP) {
return 1.0;
}
if (abs($away) > $reach) {
return 0.0;
}
$curve = sin(M_PI * $away) / (M_PI * $away);
$window = 0.5 * (1 + cos(M_PI * $away / $reach));
return $curve * $window;
}
/**
* waveOfSamples writes samples as the bytes of a wave file, which is
* the plainest way to hand sound to something that can play it. A
* caller uses this to listen to what the decoder made.
*
* @param array $samples whole numbers, one for each sample
* @param int $rate how many samples a second the sound is played at
* @return string the whole contents of a wave file
*/
public static function waveOfSamples($samples,
$rate = self::SAMPLE_RATE)
{
$sound = "";
foreach ($samples as $one) {
$sound .= pack("v", $one & 0xFFFF);
}
$head = "RIFF" . pack("V", 36 + strlen($sound)) . "WAVEfmt " .
pack("V", 16) . pack("v", 1) . pack("v", 1) .
pack("V", $rate) . pack("V", $rate * 2) . pack("v", 2) .
pack("v", 16) . "data" . pack("V", strlen($sound));
return $head . $sound;
}
}
(c) 2026 Seekquarry - Open Source Search Engine Software