Skip to content

Commit

Permalink
Fix VP9 out of order packets forwarding
Browse files Browse the repository at this point in the history
  • Loading branch information
Vittorio Palmisano committed Jan 28, 2025
1 parent 95113b6 commit 7253dbd
Show file tree
Hide file tree
Showing 3 changed files with 274 additions and 57 deletions.
104 changes: 104 additions & 0 deletions worker/include/RTC/Codecs/PayloadDescriptorHandler.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
#define MS_RTC_CODECS_PAYLOAD_DESCRIPTOR_HANDLER_HPP

#include "common.hpp"
#include "RTC/SeqManager.hpp"
#include <deque>

namespace RTC
{
Expand All @@ -14,6 +16,61 @@ namespace RTC
virtual void Dump() const = 0;
};

class PictureIdList
{
static constexpr uint16_t MaxCurrentLayerPictureIdNum{ 1000u };

public:
explicit PictureIdList()
{
}
~PictureIdList()
{
this->list.clear();
}

void Push(uint16_t pictureId, int16_t layer)
{
for (const auto& it : this->list)
{
// Layers can be changed only with ordered pictureId values.
// If pictureId is lower than the previous one, then it has rolled over the max value.
uint16_t diff = pictureId > it.first
? pictureId - it.first
: pictureId + RTC::SeqManager<uint16_t, 15>::MaxValue - it.first;

if (diff > MaxCurrentLayerPictureIdNum)
{
this->list.pop_front();
}
else
{
break;
}
}
this->list.push_back({ pictureId, layer });
}

int16_t GetLayer(uint16_t pictureId) const
{
if (this->list.size() > 1)
{
for (auto it = std::next(this->list.begin()); it != this->list.end(); ++it)
{
if (RTC::SeqManager<uint16_t, 15>::IsSeqHigherThan(it->first, pictureId))
{
return std::prev(it)->second;
}
}
}

return -1;
}

private:
std::deque<std::pair<uint16_t, int16_t>> list;
};

// Encoding context used by PayloadDescriptorHandler to properly rewrite the
// PayloadDescriptor.
class EncodingContext
Expand Down Expand Up @@ -87,13 +144,60 @@ namespace RTC
}
virtual void SyncRequired() = 0;

void SetCurrentSpatialLayer(int16_t spatialLayer, uint16_t pictureId)
{
if (this->currentSpatialLayer == spatialLayer)
{
return;
}

this->spatialLayerPictureIdList.Push(pictureId, spatialLayer);

this->currentSpatialLayer = spatialLayer;
}
void SetCurrentTemporalLayer(int16_t temporalLayer, uint16_t pictureId)
{
if (this->currentTemporalLayer == temporalLayer)
{
return;
}

this->temporalLayerPictureIdList.Push(pictureId, temporalLayer);

this->currentTemporalLayer = temporalLayer;
}
int16_t GetCurrentSpatialLayer(uint16_t pictureId) const
{
int16_t layer = this->spatialLayerPictureIdList.GetLayer(pictureId);
if (layer > -1)
{
return layer;
}

return this->currentSpatialLayer;
}
int16_t GetCurrentTemporalLayer(uint16_t pictureId) const
{
int16_t layer = this->temporalLayerPictureIdList.GetLayer(pictureId);
if (layer > -1)
{
return layer;
}

return this->currentTemporalLayer;
}

private:
Params params;
int16_t targetSpatialLayer{ -1 };
int16_t targetTemporalLayer{ -1 };
int16_t currentSpatialLayer{ -1 };
int16_t currentTemporalLayer{ -1 };
bool ignoreDtx{ false };

private:
PictureIdList spatialLayerPictureIdList;
PictureIdList temporalLayerPictureIdList;
};

class PayloadDescriptorHandler
Expand Down
117 changes: 62 additions & 55 deletions worker/src/RTC/Codecs/VP9.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,78 +242,82 @@ namespace RTC
// clang-format on

// Upgrade current spatial layer if needed.
if (context->GetTargetSpatialLayer() > context->GetCurrentSpatialLayer())
{
if (this->payloadDescriptor->isKeyFrame)
{
MS_DEBUG_DEV(
"upgrading tmpSpatialLayer from %" PRIu16 " to %" PRIu16 " (packet:%" PRIu8 ":%" PRIu8
")",
context->GetCurrentSpatialLayer(),
context->GetTargetSpatialLayer(),
packetSpatialLayer,
packetTemporalLayer);

tmpSpatialLayer = context->GetTargetSpatialLayer();
tmpTemporalLayer = 0; // Just in case.
}
}
// Downgrade current spatial layer if needed.
else if (context->GetTargetSpatialLayer() < context->GetCurrentSpatialLayer())
if (!isOldPacket)
{
// In K-SVC we must wait for a keyframe.
if (context->IsKSvc())
if (context->GetTargetSpatialLayer() > context->GetCurrentSpatialLayer())
{
if (this->payloadDescriptor->isKeyFrame)
// clang-format on
{
MS_DEBUG_DEV(
"downgrading tmpSpatialLayer from %" PRIu16 " to %" PRIu16 " (packet:%" PRIu8
":%" PRIu8 ") after keyframe (K-SVC)",
"upgrading tmpSpatialLayer from %" PRIu16 " to %" PRIu16 " (packet:%" PRIu8 ":%" PRIu8
") old:%d",
context->GetCurrentSpatialLayer(),
context->GetTargetSpatialLayer(),
packetSpatialLayer,
packetTemporalLayer);
packetTemporalLayer,
isOldPacket);

tmpSpatialLayer = context->GetTargetSpatialLayer();
tmpTemporalLayer = 0; // Just in case.
}
}
// In full SVC we do not need a keyframe.
else
// Downgrade current spatial layer if needed.
else if (context->GetTargetSpatialLayer() < context->GetCurrentSpatialLayer())
{
// clang-format off
if (
packetSpatialLayer == context->GetTargetSpatialLayer() &&
this->payloadDescriptor->e
)
// clang-format on
// In K-SVC we must wait for a keyframe.
if (context->IsKSvc())
{
MS_DEBUG_DEV(
"downgrading tmpSpatialLayer from %" PRIu16 " to %" PRIu16 " (packet:%" PRIu8
":%" PRIu8 ") without keyframe (full SVC)",
context->GetCurrentSpatialLayer(),
context->GetTargetSpatialLayer(),
packetSpatialLayer,
packetTemporalLayer);

tmpSpatialLayer = context->GetTargetSpatialLayer();
tmpTemporalLayer = 0; // Just in case.
if (this->payloadDescriptor->isKeyFrame)
// clang-format on
{
MS_DEBUG_DEV(
"downgrading tmpSpatialLayer from %" PRIu16 " to %" PRIu16 " (packet:%" PRIu8
":%" PRIu8 ") after keyframe (K-SVC)",
context->GetCurrentSpatialLayer(),
context->GetTargetSpatialLayer(),
packetSpatialLayer,
packetTemporalLayer);

tmpSpatialLayer = context->GetTargetSpatialLayer();
tmpTemporalLayer = 0; // Just in case.
}
}
// In full SVC we do not need a keyframe.
else
{
// clang-format off
if (
packetSpatialLayer == context->GetTargetSpatialLayer() &&
this->payloadDescriptor->e
)
// clang-format on
{
MS_DEBUG_DEV(
"downgrading tmpSpatialLayer from %" PRIu16 " to %" PRIu16 " (packet:%" PRIu8
":%" PRIu8 ") without keyframe (full SVC)",
context->GetCurrentSpatialLayer(),
context->GetTargetSpatialLayer(),
packetSpatialLayer,
packetTemporalLayer);

tmpSpatialLayer = context->GetTargetSpatialLayer();
tmpTemporalLayer = 0; // Just in case.
}
}
}
}

// Unless old packet filter spatial layers that are either
// Filter spatial layers that are either
// * higher than current one
// * different than the current one when KSVC is enabled and this is not a keyframe
// (interframe p bit = 1)
uint16_t tmpSpatialLayerCheck =
isOldPacket ? context->GetCurrentSpatialLayer(this->payloadDescriptor->pictureId)
: tmpSpatialLayer;
// clang-format off
if (
!isOldPacket &&
(
packetSpatialLayer > tmpSpatialLayer ||
(context->IsKSvc() && this->payloadDescriptor->p && packetSpatialLayer != tmpSpatialLayer)
)
packetSpatialLayer > tmpSpatialLayerCheck ||
(context->IsKSvc() && this->payloadDescriptor->p && packetSpatialLayer != tmpSpatialLayerCheck)
)
// clang-format on
{
Expand Down Expand Up @@ -369,12 +373,15 @@ namespace RTC
tmpTemporalLayer = context->GetTargetTemporalLayer();
}
}
}

// Filter temporal layers higher than current one.
if (packetTemporalLayer > tmpTemporalLayer)
{
return false;
}
// Filter temporal layers higher than current one.
uint16_t tmpTemporalLayerCheck =
isOldPacket ? context->GetCurrentTemporalLayer(this->payloadDescriptor->pictureId)
: tmpTemporalLayer;
if (packetTemporalLayer > tmpTemporalLayerCheck)
{
return false;
}

// Set marker bit if needed.
Expand All @@ -394,13 +401,13 @@ namespace RTC
// Update current spatial layer if needed.
if (tmpSpatialLayer != context->GetCurrentSpatialLayer())
{
context->SetCurrentSpatialLayer(tmpSpatialLayer);
context->SetCurrentSpatialLayer(tmpSpatialLayer, this->payloadDescriptor->pictureId);
}

// Update current temporal layer if needed.
if (tmpTemporalLayer != context->GetCurrentTemporalLayer())
{
context->SetCurrentTemporalLayer(tmpTemporalLayer);
context->SetCurrentTemporalLayer(tmpTemporalLayer, this->payloadDescriptor->pictureId);
}

return true;
Expand Down
Loading

0 comments on commit 7253dbd

Please sign in to comment.