Add support for receiving voice and opening up the voice api (#386)

* rework udp connection * rework audio poller * minor gateway changes * add decryption to xsalsa codec * add streams and minor changes to voice connection * concurrency is a thing, among other minor changes * better atomics * add missing KordVoice * move default audioframesender impl into separate file * sequential processing for user streams * intellij messed up packages * allow for different implementations of streams, nop by default * better handling of discord-spec rtp packets Signed-off-by: Lost <[email protected]> * kord io * replace encryption implementation * move connect to behavior * replace bounded udp channels with actual udp sockets * remove AudioPackets * update streams * connection builder fix * invert logic in voice update handler * core explicit api * fix intellij's dislike of RTPPacket * global socket shouldn't stop * external port may actually change in the global udp socket
kordlib · Oct 18, 2021 · c1e1bf5 · c1e1bf5
1 parent 38df17f
commit c1e1bf5
Show file tree

Hide file tree

Showing 40 changed files with 4,693 additions and 411 deletions.
diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts
@@ -13,7 +13,7 @@ dependencies {
     implementation(kotlin("gradle-plugin", version = "1.5.30"))
     implementation(kotlin("serialization", version = "1.5.30"))
     implementation("org.jetbrains.dokka", "dokka-gradle-plugin", "1.5.0")
-    implementation("org.jetbrains.kotlinx", "atomicfu-gradle-plugin", "0.16.1")
+    implementation("org.jetbrains.kotlinx", "atomicfu-gradle-plugin", "0.16.3")
     implementation(gradleApi())
     implementation(localGroovy())
 }
diff --git a/buildSrc/src/main/kotlin/kord-module.gradle.kts b/buildSrc/src/main/kotlin/kord-module.gradle.kts
@@ -24,6 +24,11 @@ tasks {
         onlyIf { Library.isRelease }
     }
 
+    withType<JavaCompile> {
+        sourceCompatibility = "1.8"
+        targetCompatibility = "1.8"
+    }
+
     withType<KotlinCompile> {
         kotlinOptions {
             jvmTarget = Jvm.target

diff --git a/core/src/main/kotlin/behavior/channel/BaseVoiceChannelBehavior.kt b/core/src/main/kotlin/behavior/channel/BaseVoiceChannelBehavior.kt
@@ -1,10 +1,15 @@
 package dev.kord.core.behavior.channel
 
 import dev.kord.cache.api.query
+import dev.kord.common.annotation.KordVoice
 import dev.kord.common.exception.RequestException
 import dev.kord.core.cache.data.VoiceStateData
 import dev.kord.core.cache.idEq
 import dev.kord.core.entity.VoiceState
+import dev.kord.core.entity.channel.VoiceChannel
+import dev.kord.core.exception.GatewayNotFoundException
+import dev.kord.voice.VoiceConnection
+import dev.kord.voice.VoiceConnectionBuilder
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.map
 
@@ -23,4 +28,26 @@ public interface BaseVoiceChannelBehavior : TopGuildChannelBehavior {
             .asFlow()
             .map { VoiceState(it, kord) }
 
+    /**
+     * Connect to this [VoiceChannel] and create a [VoiceConnection] for this voice session.
+     *
+     * @param builder a builder for the [VoiceConnection].
+     * @throws GatewayNotFoundException when there is no associated [dev.kord.gateway.Gateway] for the [dev.kord.core.entity.Guild] this channel is in.
+     * @throws dev.kord.voice.exception.VoiceConnectionInitializationException when there was a problem retrieving voice information from Discord.
+     * @return a [VoiceConnection] representing the connection to this [VoiceConnection].
+     */
+    @KordVoice
+    public suspend fun connect(builder: VoiceConnectionBuilder.() -> Unit): VoiceConnection {
+        val voiceConnection = VoiceConnection(
+            getGuild().gateway ?: GatewayNotFoundException.voiceConnectionGatewayNotFound(guildId),
+            kord.selfId,
+            id,
+            guildId,
+            builder
+        )
+
+        voiceConnection.connect()
+
+        return voiceConnection
+    }
 }
diff --git a/core/src/main/kotlin/entity/channel/StageVoiceChannel.kt b/core/src/main/kotlin/entity/channel/StageVoiceChannel.kt
@@ -1,17 +1,13 @@
 package dev.kord.core.entity.channel
 
-import dev.kord.common.annotation.KordVoice
 import dev.kord.common.entity.optional.getOrThrow
 import dev.kord.core.Kord
 import dev.kord.core.behavior.channel.ChannelBehavior
 import dev.kord.core.behavior.channel.GuildChannelBehavior
 import dev.kord.core.behavior.channel.StageChannelBehavior
 import dev.kord.core.cache.data.ChannelData
-import dev.kord.core.exception.GatewayNotFoundException
 import dev.kord.core.supplier.EntitySupplier
 import dev.kord.core.supplier.EntitySupplyStrategy
-import dev.kord.voice.VoiceConnection
-import dev.kord.voice.VoiceConnectionBuilder
 import java.util.*
 
 /**
@@ -57,27 +53,4 @@ public class StageChannel(
     override fun toString(): String {
         return "StageChannel(data=$data, kord=$kord, supplier=$supplier)"
     }
-
-    /**
-     * Connect to this [VoiceChannel] and create a [VoiceConnection] for this voice session.
-     *
-     * @param builder a builder for the [VoiceConnection].
-     * @throws GatewayNotFoundException when there is no associated [dev.kord.gateway.Gateway] for the [dev.kord.core.entity.Guild] this channel is in.
-     * @throws dev.kord.voice.exception.VoiceConnectionInitializationException when there was a problem retrieving voice information from Discord.
-     * @return a [VoiceConnection] representing the connection to this [VoiceConnection].
-     */
-    @KordVoice
-    public suspend fun connect(builder: VoiceConnectionBuilder.() -> Unit): VoiceConnection {
-        val voiceConnection = VoiceConnection(
-            getGuild().gateway ?: GatewayNotFoundException.voiceConnectionGatewayNotFound(guildId),
-            kord.selfId,
-            id,
-            guildId,
-            builder
-        )
-
-        voiceConnection.connect()
-
-        return voiceConnection
-    }
 }
diff --git a/core/src/main/kotlin/entity/channel/VoiceChannel.kt b/core/src/main/kotlin/entity/channel/VoiceChannel.kt
@@ -1,12 +1,10 @@
 package dev.kord.core.entity.channel
 
-import dev.kord.common.annotation.KordVoice
 import dev.kord.common.entity.optional.getOrThrow
 import dev.kord.common.exception.RequestException
 import dev.kord.core.Kord
 import dev.kord.core.behavior.channel.ChannelBehavior
 import dev.kord.core.behavior.channel.GuildChannelBehavior
-import dev.kord.core.behavior.channel.TopGuildChannelBehavior
 import dev.kord.core.behavior.channel.VoiceChannelBehavior
 import dev.kord.core.cache.data.ChannelData
 import dev.kord.core.entity.Region
@@ -86,27 +84,4 @@ public class VoiceChannel(
     override fun toString(): String {
         return "VoiceChannel(data=$data, kord=$kord, supplier=$supplier)"
     }
-
-    /**
-     * Connect to this [VoiceChannel] and create a [VoiceConnection] for this voice session.
-     *
-     * @param builder a builder for the [VoiceConnection].
-     * @throws GatewayNotFoundException when there is no associated [dev.kord.gateway.Gateway] for the [dev.kord.core.entity.Guild] this channel is in.
-     * @throws dev.kord.voice.exception.VoiceConnectionInitializationException when there was a problem retrieving voice information from Discord.
-     * @return a [VoiceConnection] representing the connection to this [VoiceConnection].
-     */
-    @KordVoice
-    public suspend fun connect(builder: VoiceConnectionBuilder.() -> Unit): VoiceConnection {
-        val voiceConnection = VoiceConnection(
-            getGuild().gateway ?: GatewayNotFoundException.voiceConnectionGatewayNotFound(guildId),
-            kord.selfId,
-            id,
-            guildId,
-            builder
-        )
-
-        voiceConnection.connect()
-
-        return voiceConnection
-    }
 }
diff --git a/settings.gradle.kts b/settings.gradle.kts
@@ -32,7 +32,6 @@ dependencyResolutionManagement {
             cache()
             common()
             tests()
-            misc()
         }
     }
 }
@@ -71,16 +70,12 @@ fun VersionCatalogBuilder.common() {
     version("kotlinx-coroutines", "1.5.2")
     alias("kotlinx-serialization").to("org.jetbrains.kotlinx", "kotlinx-serialization-json").version("1.3.0")
     alias("kotlinx-coroutines").to("org.jetbrains.kotlinx", "kotlinx-coroutines").versionRef("kotlinx-coroutines")
-    alias("kotlinx-atomicfu").to("org.jetbrains.kotlinx", "atomicfu").version("0.16.1")
+    alias("kotlinx-atomicfu").to("org.jetbrains.kotlinx", "atomicfu").version("0.16.3")
     alias("kotlin-logging").to("io.github.microutils", "kotlin-logging").version("2.0.11")
 
     bundle("common", listOf("kotlinx-serialization", "kotlinx-coroutines", "kotlinx-atomicfu", "kotlin-logging"))
 }
 
-fun VersionCatalogBuilder.misc() {
-    alias("codahale-xsalsa20poly1305").to("com.codahale", "xsalsa20poly1305").version("0.10.1")
-}
-
 fun VersionCatalogBuilder.tests() {
     val junit5 = version("junit5", "5.8.1")
 

diff --git a/voice/build.gradle.kts b/voice/build.gradle.kts
@@ -8,9 +8,16 @@ dependencies {
     api(common)
     api(gateway)
 
-    implementation(libs.codahale.xsalsa20poly1305)
     api(libs.ktor.client.json)
     api(libs.ktor.client.serialization)
     api(libs.ktor.client.cio)
     api(libs.ktor.network)
 }
+
+// by convention, java classes (TweetNaclFast) should be in their own java source.
+// however, this breaks atomicfu.
+// to work around it we just make the kotlin src directory also a java src directory.
+// this can be removed when https://github.com/Kotlin/kotlinx.atomicfu/commit/fe0950adcf0da67cd074503c2a78467656c5aa0f is released.
+sourceSets.main {
+    java.srcDirs("src/main/java", "src/main/kotlin")
+}
diff --git a/voice/src/main/kotlin/AudioFrame.kt b/voice/src/main/kotlin/AudioFrame.kt
@@ -3,7 +3,7 @@ package dev.kord.voice
 import dev.kord.common.annotation.KordVoice
 
 /**
- * A frame of 20ms Opus-encoded audio data.
+ * A frame of 20ms Opus-encoded 48k stereo audio data.
  */
 @KordVoice
 @JvmInline

diff --git a/voice/src/main/kotlin/AudioProvider.kt b/voice/src/main/kotlin/AudioProvider.kt
@@ -21,7 +21,7 @@ fun interface AudioProvider {
      *
      * @return the frame of audio.
      */
-    fun provide(): AudioFrame?
+    suspend fun provide(): AudioFrame?
 
     /**
      * Polls [AudioFrame]s into the [frames] channel at an appropriate interval. Suspends until the coroutine scope is cancelled.
@@ -33,13 +33,14 @@ fun interface AudioProvider {
         var nextFrameTimestamp = mark.elapsedNow().inWholeNanoseconds
 
         while (isActive) {
-            nextFrameTimestamp += Duration.milliseconds(20).inWholeNanoseconds
-            delayUntilNextFrameTimestamp(mark, nextFrameTimestamp)
             frames.send(provide())
+
+            nextFrameTimestamp += 20_000_000
+            delayUntilNextFrameTimestamp(mark.elapsedNow().inWholeNanoseconds, nextFrameTimestamp)
         }
     }
 }
 
-private suspend inline fun delayUntilNextFrameTimestamp(mark: TimeMark, nextFrameTimestamp: Long) {
-    delay(Duration.nanoseconds(max(0, nextFrameTimestamp - mark.elapsedNow().inWholeNanoseconds)).inWholeMilliseconds)
+private suspend inline fun delayUntilNextFrameTimestamp(now: Long, nextFrameTimestamp: Long) {
+    delay(max(0, nextFrameTimestamp - now) / 1_000_000)
 }
diff --git a/voice/src/main/kotlin/EncryptionMode.kt b/voice/src/main/kotlin/EncryptionMode.kt
@@ -14,12 +14,12 @@ enum class EncryptionMode {
     XSalsa20Poly1305Suffix,
 
     @SerialName("xsalsa20_poly1305_lite")
-    XSalsa20Poly1306Lite,
+    XSalsa20Poly1305Lite,
 
     @SerialName("xsalsa20_poly1305_lite_rtpsize")
     XSalsa20Poly1305LiteRtpsize,
 
-    // video stuff... unused. though required to allow for serialization of ready
+    // video/unreleased-audio stuff... unused. though required to allow for serialization of ready
     @SerialName("aead_aes256_gcm_rtpsize")
     AeadAes256GcmRtpsize,
 

diff --git a/voice/src/main/kotlin/FrameInterceptor.kt b/voice/src/main/kotlin/FrameInterceptor.kt
@@ -2,7 +2,7 @@ package dev.kord.voice
 
 import dev.kord.common.annotation.KordVoice
 import dev.kord.gateway.Gateway
-import dev.kord.voice.gateway.Speaking
+import dev.kord.voice.gateway.SendSpeaking
 import dev.kord.voice.gateway.VoiceGateway
 import kotlin.properties.Delegates
 
@@ -17,12 +17,12 @@ import kotlin.properties.Delegates
 data class FrameInterceptorContext(
     val gateway: Gateway,
     val voiceGateway: VoiceGateway,
-    val ssrc: Int,
+    val ssrc: UInt,
 )
 
 @KordVoice
-internal class FrameInterceptorContextBuilder(var gateway: Gateway, var voiceGateway: VoiceGateway) {
-    var ssrc: Int by Delegates.notNull()
+class FrameInterceptorContextBuilder(var gateway: Gateway, var voiceGateway: VoiceGateway) {
+    var ssrc: UInt by Delegates.notNull()
 
     fun build() = FrameInterceptorContext(gateway, voiceGateway, ssrc)
 }
@@ -32,51 +32,56 @@ internal inline fun FrameInterceptorContext(gateway: Gateway, voiceGateway: Voic
     FrameInterceptorContextBuilder(gateway, voiceGateway).apply(builder).build()
 
 /**
- * A interceptor for audio frames before they are sent as packets.
+ * An interceptor for audio frames before they are sent as packets.
  *
  * @see DefaultFrameInterceptor
  */
 @KordVoice
 fun interface FrameInterceptor {
-    suspend fun intercept(audioFrame: AudioFrame?): AudioFrame?
+    suspend fun intercept(frame: AudioFrame?): AudioFrame?
 }
 
+private const val FRAMES_OF_SILENCE_TO_PLAY = 5
+
 /**
  * The default implementation for [FrameInterceptor].
  * Any custom implementation should extend this and call the super [intercept] method, or else
  * the speaking flags will not be sent!
  *
- * @param connection the voice connection.
+ * @param context the context for this interceptor.
  * @param speakingState the speaking state that will be used when there is audio data to be sent. By default, it is microphone-only.
  */
 @KordVoice
 open class DefaultFrameInterceptor(
     protected val context: FrameInterceptorContext,
     private val speakingState: SpeakingFlags = SpeakingFlags { +SpeakingFlag.Microphone }
 ) : FrameInterceptor {
+    private val voiceGateway = context.voiceGateway
+
     private var framesOfSilence = 5
     private var isSpeaking = false
 
-    override suspend fun intercept(audioFrame: AudioFrame?): AudioFrame? = with(context) {
-        var frame: AudioFrame? = null
+    private val nowSpeaking = SendSpeaking(speakingState, 0, context.ssrc)
+    private val notSpeaking = SendSpeaking(SpeakingFlags(0), 0, context.ssrc)
 
-        if (audioFrame != null || framesOfSilence > 0) {
-            if (!isSpeaking && audioFrame != null) {
+    override suspend fun intercept(frame: AudioFrame?): AudioFrame? {
+        if (frame != null || framesOfSilence > 0) { // is there something to process
+            if (!isSpeaking && frame != null) { // if there is audio make sure we are speaking
                 isSpeaking = true
-                voiceGateway.send(Speaking(speakingState, 0, ssrc))
+                voiceGateway.send(nowSpeaking)
             }
 
-            frame = audioFrame ?: AudioFrame.SILENCE
-
-            if (audioFrame == null) {
-                framesOfSilence--
-                if (framesOfSilence == 0) {
+            if (frame == null) { // if we don't have audio then make sure we know that we are sending a frame of silence
+                if (--framesOfSilence == 0) { // we're done with frames of silence if we hit zero
                     isSpeaking = false
-                    voiceGateway.send(Speaking(SpeakingFlags(0), 0, ssrc))
+                    voiceGateway.send(notSpeaking)
                 }
-            } else {
-                framesOfSilence = 5
             }
+            else if (framesOfSilence != FRAMES_OF_SILENCE_TO_PLAY) {
+                framesOfSilence = FRAMES_OF_SILENCE_TO_PLAY // we're playing audio, lets reset the frames of silence.
+            }
+
+            return frame ?: AudioFrame.SILENCE
         }
 
         return frame