TextureLoader feature.

HaishinKit · Jan 14, 2024 · 02d0949 · 02d0949
1 parent 3535ec9
commit 02d0949
Show file tree

Hide file tree

Showing 52 changed files with 747 additions and 546 deletions.
diff --git a/app/src/main/java/com/haishinkit/app/CameraTabFragment.kt b/app/src/main/java/com/haishinkit/app/CameraTabFragment.kt
@@ -40,6 +40,7 @@ class CameraTabFragment : Fragment(), IEventListener {
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
+
         activity?.let {
             val permissionCheck = ContextCompat.checkSelfPermission(it, Manifest.permission.CAMERA)
             if (permissionCheck != PackageManager.PERMISSION_GRANTED) {

diff --git a/app/src/main/java/com/haishinkit/app/MainActivity.kt b/app/src/main/java/com/haishinkit/app/MainActivity.kt
@@ -6,19 +6,12 @@ import android.view.MenuItem
 import androidx.appcompat.app.AppCompatActivity
 import androidx.fragment.app.Fragment
 import com.google.android.material.bottomnavigation.BottomNavigationView
-import com.haishinkit.graphics.PixelTransformFactory
-import com.haishinkit.vulkan.VkPixelTransform
 
 class MainActivity : AppCompatActivity(), BottomNavigationView.OnNavigationItemSelectedListener {
     private var fragment: Fragment? = null
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
-
-        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
-            PixelTransformFactory.registerPixelTransform(VkPixelTransform::class)
-        }
-
         setContentView(R.layout.activity_main)
 
         val navigation = findViewById<BottomNavigationView>(R.id.navigation)
@@ -34,15 +27,19 @@ class MainActivity : AppCompatActivity(), BottomNavigationView.OnNavigationItemS
             R.id.navigation_home -> {
                 fragment = CameraTabFragment.newInstance()
             }
+
             R.id.navigation_mediaprojection -> {
                 fragment = MediaProjectionTabFragment.newInstance()
             }
+
             R.id.navigation_playback -> {
                 fragment = PlaybackTabFragment.newInstance()
             }
+
             R.id.navigation_dashboard -> {
                 fragment = PreferenceTagFragment.newInstance()
             }
+
             else -> {
             }
         }

diff --git a/app/src/main/java/com/haishinkit/app/MediaProjectionService.kt b/app/src/main/java/com/haishinkit/app/MediaProjectionService.kt
@@ -47,8 +47,8 @@ class MediaProjectionService : Service(), IEventListener {
                 MSG_SET_VIDEO_EFFECT -> {
                     if (msg.obj is LanczosVideoEffect) {
                         val lanczosVideoEffect = msg.obj as LanczosVideoEffect
-                        lanczosVideoEffect.texelWidth = videoSource.resolution.width.toFloat()
-                        lanczosVideoEffect.texelHeight = videoSource.resolution.height.toFloat()
+                        lanczosVideoEffect.texelWidth = videoSource.size.width.toFloat()
+                        lanczosVideoEffect.texelHeight = videoSource.size.height.toFloat()
                         stream.videoEffect = lanczosVideoEffect
                         return
                     }
@@ -95,8 +95,8 @@ class MediaProjectionService : Service(), IEventListener {
                 metrics
             )
             stream.attachVideo(source)
-            stream.videoSetting.width = source.resolution.width shr 2
-            stream.videoSetting.height = source.resolution.height shr 2
+            stream.videoSetting.width = source.size.width shr 2
+            stream.videoSetting.height = source.size.height shr 2
             videoSource = source
             Log.e(TAG, "${stream.videoSetting.width}:${stream.videoSetting.height}")
         }

diff --git a/app/src/main/java/com/haishinkit/app/Preference.kt b/app/src/main/java/com/haishinkit/app/Preference.kt
@@ -3,7 +3,7 @@ package com.haishinkit.app
 data class Preference(var rtmpURL: String, var streamName: String) {
     companion object {
         var shared = Preference(
-            "rtmp://test:[email protected].3/live",
+            "rtmp://test:[email protected].4/live",
             "live"
         )
 

diff --git a/haishinkit/src/main/java/com/haishinkit/amf/Amf0Deserializer.kt b/haishinkit/src/main/java/com/haishinkit/amf/Amf0Deserializer.kt
@@ -19,18 +19,22 @@ internal class Amf0Deserializer(private val buffer: ByteBuffer) {
                     buffer.position(buffer.position() - 1)
                     return double
                 }
+
                 Amf0Marker.BOOL.rawValue -> {
                     buffer.position(buffer.position() - 1)
                     return boolean
                 }
+
                 Amf0Marker.STRING.rawValue -> {
                     buffer.position(buffer.position() - 1)
                     return string
                 }
+
                 Amf0Marker.OBJECT.rawValue -> {
                     buffer.position(buffer.position() - 1)
                     return map
                 }
+
                 Amf0Marker.MOVIECLIP.rawValue -> throw UnsupportedOperationException()
                 Amf0Marker.NULL.rawValue -> return null
                 Amf0Marker.UNDEFINED.rawValue -> return AsUndefined.instance
@@ -39,25 +43,30 @@ internal class Amf0Deserializer(private val buffer: ByteBuffer) {
                     buffer.position(buffer.position() - 1)
                     return list
                 }
+
                 Amf0Marker.OBJECTEND.rawValue -> throw UnsupportedOperationException()
                 Amf0Marker.STRICTARRAY.rawValue -> {
                     buffer.position(buffer.position() - 1)
                     return objects
                 }
+
                 Amf0Marker.DATE.rawValue -> {
                     buffer.position(buffer.position() - 1)
                     return date
                 }
+
                 Amf0Marker.LONGSTRING.rawValue -> {
                     buffer.position(buffer.position() - 1)
                     return string
                 }
+
                 Amf0Marker.UNSUPPORTED.rawValue -> throw UnsupportedOperationException()
                 Amf0Marker.RECORDSET.rawValue -> throw UnsupportedOperationException()
                 Amf0Marker.XMLDOCUMENT.rawValue -> {
                     buffer.position(buffer.position() - 1)
                     return xmlDocument
                 }
+
                 Amf0Marker.TYPEDOBJECT.rawValue -> throw UnsupportedOperationException()
                 Amf0Marker.AVMPLUSH.rawValue -> throw UnsupportedOperationException()
                 else -> {
@@ -91,6 +100,7 @@ internal class Amf0Deserializer(private val buffer: ByteBuffer) {
             when (marker) {
                 Amf0Marker.STRING.rawValue, Amf0Marker.LONGSTRING.rawValue -> {
                 }
+
                 else -> throw IllegalFormatFlagsException(marker.toString())
             }
             return getString(Amf0Marker.STRING.rawValue == marker)

diff --git a/haishinkit/src/main/java/com/haishinkit/codec/Codec.kt b/haishinkit/src/main/java/com/haishinkit/codec/Codec.kt
@@ -150,6 +150,7 @@ abstract class Codec(private val mime: String) : Running {
                             value?.let { codec?.setInputSurface(it) }
                         }
                     }
+
                     Mode.DECODE -> {
                         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
                             value?.let { codec?.setOutputSurface(it) }

diff --git a/haishinkit/src/main/java/com/haishinkit/codec/VideoCodec.kt b/haishinkit/src/main/java/com/haishinkit/codec/VideoCodec.kt
@@ -1,19 +1,20 @@
 package com.haishinkit.codec
 
 import android.content.res.AssetManager
-import android.media.MediaCodecInfo
 import android.media.MediaCodec
+import android.media.MediaCodecInfo
 import android.media.MediaFormat
 import android.os.Build
 import android.os.Bundle
 import android.util.Size
 import com.haishinkit.graphics.PixelTransform
 import com.haishinkit.graphics.PixelTransformFactory
 import com.haishinkit.graphics.VideoGravity
+import com.haishinkit.media.VideoSource
 import com.haishinkit.util.FeatureUtil
 import kotlin.properties.Delegates
 
-class VideoCodec : Codec(MIME) {
+class VideoCodec(val videoSource: VideoSource) : Codec(MIME) {
     @Suppress("unused")
     data class Setting(private val codec: VideoCodec? = null) : Codec.Setting(codec) {
         /**
@@ -137,6 +138,7 @@ class VideoCodec : Codec(MIME) {
         PixelTransformFactory().create().apply {
             videoGravity = DEFAULT_VIDEO_GRAVITY
             frameRate = DEFAULT_FRAME_RATE
+            videoSource = this@VideoCodec.videoSource
         }
     }
 

diff --git a/haishinkit/src/main/java/com/haishinkit/event/EventDispatcher.kt b/haishinkit/src/main/java/com/haishinkit/event/EventDispatcher.kt
@@ -4,7 +4,7 @@ import androidx.core.util.Pools
 import java.util.Collections
 import java.util.concurrent.ConcurrentHashMap
 
-open class EventDispatcher(private val target: IEventDispatcher?) : IEventDispatcher {
+open class EventDispatcher(private val target: IEventDispatcher? = null) : IEventDispatcher {
     private val pool = Pools.SynchronizedPool<Event>(MAX_POOL_SIZE)
     private val listeners = ConcurrentHashMap<String, MutableList<IEventListener>>()
 

diff --git a/haishinkit/src/main/java/com/haishinkit/flv/FlvWriter.kt b/haishinkit/src/main/java/com/haishinkit/flv/FlvWriter.kt
@@ -58,9 +58,11 @@ class FlvWriter : Closeable {
                 (message is RtmpAudioMessage) -> {
                     audioTimestamp
                 }
+
                 (message is RtmpVideoMessage) -> {
                     videoTimestamp
                 }
+
                 else -> {
                     0
                 }

diff --git a/haishinkit/src/main/java/com/haishinkit/graphics/GraphicsContext.kt b/haishinkit/src/main/java/com/haishinkit/graphics/GraphicsContext.kt
@@ -0,0 +1,5 @@
+package com.haishinkit.graphics
+
+object GraphicsContext {
+    val textureRenderer by lazy { TextureRenderer.create() }
+}
diff --git a/haishinkit/src/main/java/com/haishinkit/graphics/PixelTransform.kt b/haishinkit/src/main/java/com/haishinkit/graphics/PixelTransform.kt
@@ -5,6 +5,7 @@ import android.graphics.Bitmap
 import android.util.Size
 import android.view.Surface
 import com.haishinkit.graphics.effect.VideoEffect
+import com.haishinkit.media.VideoSource
 
 /**
  * The PixelTransform interface provides some graphics operations.
@@ -30,11 +31,6 @@ interface PixelTransform {
      */
     var resampleFilter: ResampleFilter
 
-    /**
-     * Specifies the imageOrientation that describe the image orientation.
-     */
-    var imageOrientation: ImageOrientation
-
     /**
      * Specifies the deviceOrientation that describe the physical orientation of the device.
      */
@@ -60,17 +56,20 @@ interface PixelTransform {
      */
     var frameRate: Int
 
+    /**
+     * Specifies the video source.
+     */
+    var videoSource: VideoSource?
+
     /**
      * Reads the pixels of a displayed image.
      */
     fun readPixels(lambda: ((bitmap: Bitmap?) -> Unit))
 
-    fun createInputSurface(
-        width: Int,
-        height: Int,
-        format: Int,
-        lambda: ((surface: Surface) -> Unit)
-    )
+    /**
+     * Enter frame.
+     */
+    fun doFrame(frameTimeNanos: Long)
 
     /**
      * Disposes the pixelTransform for the memory management.

diff --git a/haishinkit/src/main/java/com/haishinkit/graphics/PixelTransformFactory.kt b/haishinkit/src/main/java/com/haishinkit/graphics/PixelTransformFactory.kt
@@ -1,18 +1,18 @@
 package com.haishinkit.graphics
 
-import com.haishinkit.graphics.gles.GlThreadPixelTransform
+import com.haishinkit.graphics.gles.ThreadPixelTransform
 import kotlin.reflect.KClass
 
 class PixelTransformFactory {
     fun create(): PixelTransform {
         if (pixelTransforms.isEmpty()) {
-            return GlThreadPixelTransform()
+            return ThreadPixelTransform()
         }
         return try {
             val pixelTransformClass = pixelTransforms.first()
             pixelTransformClass.java.newInstance() as PixelTransform
         } catch (e: Exception) {
-            return GlThreadPixelTransform()
+            return ThreadPixelTransform()
         }
     }
 

diff --git a/haishinkit/src/main/java/com/haishinkit/graphics/ResampleFilter.kt b/haishinkit/src/main/java/com/haishinkit/graphics/ResampleFilter.kt
@@ -2,6 +2,7 @@ package com.haishinkit.graphics
 
 import javax.microedition.khronos.opengles.GL10
 
+@Suppress("UNUSED")
 enum class ResampleFilter(val rawValue: Int, val glValue: Int) {
     LINEAR(0, GL10.GL_LINEAR),
     NEAREST(1, GL10.GL_NEAREST)

diff --git a/haishinkit/src/main/java/com/haishinkit/graphics/ScheduledFpsController.kt b/haishinkit/src/main/java/com/haishinkit/graphics/ScheduledFpsController.kt
@@ -13,22 +13,27 @@ internal class ScheduledFpsController : FpsController {
                     field = 60
                     updatePatternIdx = 5
                 }
+
                 (48 <= value) -> {
                     field = 48
                     updatePatternIdx = 4
                 }
+
                 (30 <= value) -> {
                     field = 30
                     updatePatternIdx = 3
                 }
+
                 (25 <= value) -> {
                     field = 25
                     updatePatternIdx = 2
                 }
+
                 (24 <= value) -> {
                     field = 24
                     updatePatternIdx = 1
                 }
+
                 else -> {
                     field = 15
                     updatePatternIdx = 0

diff --git a/haishinkit/src/main/java/com/haishinkit/graphics/Texture.kt b/haishinkit/src/main/java/com/haishinkit/graphics/Texture.kt
@@ -0,0 +1,19 @@
+package com.haishinkit.graphics
+
+import android.util.Size
+import android.view.Surface
+
+interface Texture {
+    enum class ReadyState {
+        CREATED, RELEASING, RELEASED
+    }
+
+    val id: Int
+    val surface: Surface?
+    val readyState: ReadyState
+    val imageExtent: Size
+    var imageOrientation: ImageOrientation
+    fun update()
+    fun release()
+    fun dispose()
+}
diff --git a/haishinkit/src/main/java/com/haishinkit/graphics/TextureRenderer.kt b/haishinkit/src/main/java/com/haishinkit/graphics/TextureRenderer.kt
@@ -0,0 +1,44 @@
+package com.haishinkit.graphics
+
+import com.haishinkit.event.IEventDispatcher
+import kotlin.reflect.KClass
+
+interface TextureRenderer : IEventDispatcher {
+    fun createTexture(width: Int, height: Int, format: Int): Texture
+    fun registerTexture(texture: Texture?)
+
+    companion object {
+        const val FRAME_UPDATED_EVENT = "frameUpdatedEvent"
+
+        private var textureLoaders: MutableList<KClass<*>> = mutableListOf()
+
+        internal fun create(): TextureRenderer {
+            if (textureLoaders.isEmpty()) {
+                return com.haishinkit.graphics.gles.TextureRenderer()
+            }
+            return try {
+                val textureLoaderClass = textureLoaders.first()
+                textureLoaderClass.java.newInstance() as TextureRenderer
+            } catch (e: Exception) {
+                com.haishinkit.graphics.gles.TextureRenderer()
+            }
+        }
+
+        fun <T : TextureRenderer> registerTextureLoader(clazz: KClass<T>) {
+            for (i in 0 until textureLoaders.size) {
+                if (textureLoaders[i] == clazz) {
+                    return
+                }
+            }
+            textureLoaders.add(clazz)
+        }
+
+        fun <T : TextureRenderer> unregisterTextureLoader(clazz: KClass<T>) {
+            for (i in (0 until textureLoaders.size).reversed()) {
+                if (textureLoaders[i] == clazz) {
+                    textureLoaders.removeAt(i)
+                }
+            }
+        }
+    }
+}