Handle OpenGL rendering on frontend

This substantially improves performance when using the OpenGL renderer

app/CMakeLists.txt

@@ -49,6 +49,7 @@ add_library(
         src/main/cpp/UriFileHandler.cpp
         src/main/cpp/JniEnvHandler.cpp
         src/main/cpp/MelonDSAndroidCameraHandler.cpp
+        src/main/cpp/AndroidFrameRenderedCallback.cpp
         src/main/cpp/AndroidRACallback.cpp
         src/main/cpp/RAAchievementMapper.cpp
 )

app/proguard-rules.pro

@@ -42,6 +42,7 @@
 }
 -keep interface me.magnum.melonds.common.camera.DSiCameraSource { *; }
 -keep interface me.magnum.melonds.common.RetroAchievementsCallback { *; }
+-keep interface me.magnum.melonds.ui.emulator.EmulatorFrameRenderedListener { *; }
 
 # Migration fields. These rules are required for migrations to work properly
 -keep,allowobfuscation class me.magnum.melonds.migrations.legacy.** { *; }

app/src/main/cpp/AndroidFrameRenderedCallback.cpp

@@ -0,0 +1,16 @@
+#include "AndroidFrameRenderedCallback.h"
+
+AndroidFrameRenderedCallback::AndroidFrameRenderedCallback(JniEnvHandler* jniEnvHandler, jobject androidFrameRenderedListener)
+{
+    this->jniEnvHandler = jniEnvHandler;
+    this->androidFrameRenderedListener = androidFrameRenderedListener;
+}
+
+void AndroidFrameRenderedCallback::onFrameRendered(long syncFence, int textureId)
+{
+    JNIEnv* env = this->jniEnvHandler->getCurrentThreadEnv();
+
+    jclass listenerClass = env->GetObjectClass(this->androidFrameRenderedListener);
+    jmethodID onFrameRenderedMethod = env->GetMethodID(listenerClass, "onFrameRendered", "(JI)V");
+    env->CallVoidMethod(this->androidFrameRenderedListener, onFrameRenderedMethod, syncFence, textureId);
+}

app/src/main/cpp/AndroidFrameRenderedCallback.h

@@ -0,0 +1,20 @@
+#ifndef ANDROIDFRAMERENDEREDCALLBACK_H
+#define ANDROIDFRAMERENDEREDCALLBACK_H
+
+#include "JniEnvHandler.h"
+#include <FrameRenderedCallback.h>
+#include <jni.h>
+
+class AndroidFrameRenderedCallback : public FrameRenderedCallback
+{
+private:
+    JniEnvHandler* jniEnvHandler;
+    jobject androidFrameRenderedListener;
+
+public:
+    AndroidFrameRenderedCallback(JniEnvHandler* jniEnvHandler, jobject androidFrameRenderedListener);
+    void onFrameRendered(long syncFence, int textureId);
+};
+
+
+#endif //ANDROIDFRAMERENDEREDCALLBACK_H

app/src/main/cpp/MelonDSAndroidJNI.cpp

@@ -12,6 +12,7 @@
 #include "UriFileHandler.h"
 #include "JniEnvHandler.h"
 #include "AndroidRACallback.h"
+#include "AndroidFrameRenderedCallback.h"
 #include "MelonDSAndroidInterface.h"
 #include "MelonDSAndroidConfiguration.h"
 #include "MelonDSAndroidCameraHandler.h"
@@ -46,29 +47,32 @@ bool isFastForwardEnabled = false;
 jobject globalAssetManager;
 jobject globalCameraManager;
 jobject androidRaCallback;
+jobject androidFrameRenderListener;
 MelonDSAndroidCameraHandler* androidCameraHandler;
 AndroidRACallback* raCallback;
+AndroidFrameRenderedCallback* frameRenderedCallback;
 
 extern "C"
 {
 JNIEXPORT void JNICALL
-Java_me_magnum_melonds_MelonEmulator_setupEmulator(JNIEnv* env, jobject thiz, jobject emulatorConfiguration, jobject javaAssetManager, jobject cameraManager, jobject retroAchievementsCallback, jobject textureBuffer)
+Java_me_magnum_melonds_MelonEmulator_setupEmulator(JNIEnv* env, jobject thiz, jobject emulatorConfiguration, jobject javaAssetManager, jobject cameraManager, jobject retroAchievementsCallback, jobject frameRenderListener, jobject screenshotBuffer, jlong glContext)
 {
     MelonDSAndroid::EmulatorConfiguration finalEmulatorConfiguration = MelonDSAndroidConfiguration::buildEmulatorConfiguration(env, emulatorConfiguration);
     fastForwardSpeedMultiplier = finalEmulatorConfiguration.fastForwardSpeedMultiplier;
 
     globalAssetManager = env->NewGlobalRef(javaAssetManager);
     globalCameraManager = env->NewGlobalRef(cameraManager);
     androidRaCallback = env->NewGlobalRef(retroAchievementsCallback);
+    androidFrameRenderListener = env->NewGlobalRef(frameRenderListener);
 
     AAssetManager* assetManager = AAssetManager_fromJava(env, globalAssetManager);
     androidCameraHandler = new MelonDSAndroidCameraHandler(jniEnvHandler, globalCameraManager);
     raCallback = new AndroidRACallback(jniEnvHandler, androidRaCallback);
-
-    u32* textureBufferPointer = (u32*) env->GetDirectBufferAddress(textureBuffer);
+    frameRenderedCallback = new AndroidFrameRenderedCallback(jniEnvHandler, androidFrameRenderListener);
+    u32* screenshotBufferPointer = (u32*) env->GetDirectBufferAddress(screenshotBuffer);
 
     MelonDSAndroid::setConfiguration(finalEmulatorConfiguration);
-    MelonDSAndroid::setup(assetManager, androidCameraHandler, raCallback, textureBufferPointer, true);
+    MelonDSAndroid::setup(assetManager, androidCameraHandler, raCallback, frameRenderedCallback, screenshotBufferPointer, glContext, true);
     paused = false;
 }
 
@@ -415,13 +419,16 @@ Java_me_magnum_melonds_MelonEmulator_stopEmulation(JNIEnv* env, jobject thiz)
     env->DeleteGlobalRef(globalAssetManager);
     env->DeleteGlobalRef(globalCameraManager);
     env->DeleteGlobalRef(androidRaCallback);
+    env->DeleteGlobalRef(androidFrameRenderListener);
 
     globalAssetManager = nullptr;
     globalCameraManager = nullptr;
     androidRaCallback = nullptr;
+    androidFrameRenderListener = nullptr;
 
     delete androidCameraHandler;
     delete raCallback;
+    delete frameRenderedCallback;
 }
 
 JNIEXPORT void JNICALL
@@ -462,12 +469,11 @@ Java_me_magnum_melonds_MelonEmulator_setFastForwardEnabled(JNIEnv* env, jobject
 }
 
 JNIEXPORT void JNICALL
-Java_me_magnum_melonds_MelonEmulator_updateEmulatorConfiguration(JNIEnv* env, jobject thiz, jobject emulatorConfiguration, jobject frameBuffer)
+Java_me_magnum_melonds_MelonEmulator_updateEmulatorConfiguration(JNIEnv* env, jobject thiz, jobject emulatorConfiguration)
 {
     MelonDSAndroid::EmulatorConfiguration newConfiguration = MelonDSAndroidConfiguration::buildEmulatorConfiguration(env, emulatorConfiguration);
-    u32* frameBufferPointer = (u32*) env->GetDirectBufferAddress(frameBuffer);
 
-    MelonDSAndroid::updateEmulatorConfiguration(newConfiguration, frameBufferPointer);
+    MelonDSAndroid::updateEmulatorConfiguration(newConfiguration);
     fastForwardSpeedMultiplier = newConfiguration.fastForwardSpeedMultiplier;
 
     if (isFastForwardEnabled) {

app/src/main/java/me/magnum/melonds/MelonEmulator.kt

@@ -8,6 +8,7 @@ import me.magnum.melonds.domain.model.EmulatorConfiguration
 import me.magnum.melonds.domain.model.Input
 import me.magnum.melonds.domain.model.retroachievements.RASimpleAchievement
 import me.magnum.melonds.common.RetroAchievementsCallback
+import me.magnum.melonds.ui.emulator.EmulatorFrameRenderedListener
 import me.magnum.melonds.ui.emulator.rewind.model.RewindSaveState
 import me.magnum.melonds.ui.emulator.rewind.model.RewindWindow
 import java.nio.ByteBuffer
@@ -43,7 +44,15 @@ object MelonEmulator {
         MEMORY_EXPANSION,
     }
 
-	external fun setupEmulator(emulatorConfiguration: EmulatorConfiguration, assetManager: AssetManager?, dsiCameraSource: DSiCameraSource?, retroAchievementsCallback: RetroAchievementsCallback, textureBuffer: ByteBuffer)
+	external fun setupEmulator(
+        emulatorConfiguration: EmulatorConfiguration,
+        assetManager: AssetManager?,
+        dsiCameraSource: DSiCameraSource?,
+        retroAchievementsCallback: RetroAchievementsCallback,
+        frameRenderedListener: EmulatorFrameRenderedListener,
+        screenshotBuffer: ByteBuffer,
+        glContext: Long,
+    )
 
     external fun setupCheats(cheats: Array<Cheat>)
 
@@ -119,5 +128,5 @@ object MelonEmulator {
 
     external fun setFastForwardEnabled(enabled: Boolean)
 
-    external fun updateEmulatorConfiguration(emulatorConfiguration: EmulatorConfiguration, frameBuffer: ByteBuffer)
+    external fun updateEmulatorConfiguration(emulatorConfiguration: EmulatorConfiguration)
 }

app/src/main/java/me/magnum/melonds/common/opengl/Shader.kt

@@ -2,7 +2,10 @@ package me.magnum.melonds.common.opengl
 
 import android.opengl.GLES20
 
-class Shader(private val programId: Int) {
+class Shader(
+    private val programId: Int,
+    val textureFiltering: Int,
+) {
     val uniformMvp: Int
     val attribUv: Int
     val attribPos: Int

app/src/main/java/me/magnum/melonds/common/opengl/ShaderFactory.kt

@@ -4,10 +4,16 @@ import android.opengl.GLES20
 
 object ShaderFactory {
     fun createShaderProgram(source: ShaderProgramSource): Shader {
-        return createShaderProgram(source.vertexShaderSource, source.fragmentShaderSource)
+        val shaderProgram = createShaderProgram(source.vertexShaderSource, source.fragmentShaderSource)
+        val textureFilter = when (source.textureFiltering) {
+            ShaderProgramSource.TextureFiltering.NEAREST -> GLES20.GL_NEAREST
+            ShaderProgramSource.TextureFiltering.LINEAR -> GLES20.GL_LINEAR
+        }
+
+        return Shader(shaderProgram, textureFilter)
     }
 
-    private fun createShaderProgram(vertexShader: String, fragmentShader: String): Shader {
+    private fun createShaderProgram(vertexShader: String, fragmentShader: String): Int {
         val program = GLES20.glCreateProgram()
         GLES20.glAttachShader(program, createShader(GLES20.GL_VERTEX_SHADER, vertexShader))
         GLES20.glAttachShader(program, createShader(GLES20.GL_FRAGMENT_SHADER, fragmentShader))
@@ -19,7 +25,7 @@ object ShaderFactory {
             System.err.println(GLES20.glGetProgramInfoLog(program))
         }
 
-        return Shader(program)
+        return program
     }
 
     private fun createShader(shaderType: Int, code: String): Int {

app/src/main/java/me/magnum/melonds/common/runtime/ScreenshotFrameBufferProvider.kt

@@ -0,0 +1,53 @@
+package me.magnum.melonds.common.runtime
+
+import android.graphics.Bitmap
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+
+class ScreenshotFrameBufferProvider {
+
+    companion object {
+        private const val SCREEN_WIDTH = 256
+        private const val SCREEN_HEIGHT = 384
+    }
+
+    private var screenshotBuffer: ByteBuffer? = null
+
+    fun frameBuffer(): ByteBuffer {
+        return ensureBufferIsReady()
+    }
+
+    fun getScreenshot(): Bitmap {
+        val frameBuffer = ensureBufferIsReady()
+
+        return Bitmap.createBitmap(SCREEN_WIDTH, SCREEN_HEIGHT, Bitmap.Config.ARGB_8888).apply {
+            for (x in 0 until SCREEN_WIDTH) {
+                for (y in 0 until SCREEN_HEIGHT) {
+                    val pixelPosition = (y * SCREEN_WIDTH + x) * 4
+                    // There's no need to do a manual pixel format conversion. Since getInt() uses the buffer's byte order, which is little endian, it will automatically
+                    // convert the internal BGRA format into the ARGB format, which is what we need to build the bitmap
+                    val argbPixel = frameBuffer.getInt(pixelPosition)
+                    setPixel(x, y, argbPixel)
+                }
+            }
+        }
+    }
+
+    fun clearBuffer() {
+        screenshotBuffer?.let { buffer ->
+            buffer.position(0)
+            repeat(buffer.capacity() / 4) {
+                buffer.putInt(0xFF000000.toInt())
+            }
+        }
+    }
+
+    private fun ensureBufferIsReady(): ByteBuffer {
+        if (screenshotBuffer != null) {
+            return screenshotBuffer!!
+        }
+
+        screenshotBuffer = ByteBuffer.allocateDirect(SCREEN_WIDTH * SCREEN_HEIGHT * 4).order(ByteOrder.nativeOrder())
+        return screenshotBuffer!!
+    }
+}

app/src/main/java/me/magnum/melonds/di/EmulatorRuntimeModule.kt

@@ -14,7 +14,7 @@ import me.magnum.melonds.common.PermissionHandler
 import me.magnum.melonds.common.camera.BlackDSiCameraSource
 import me.magnum.melonds.common.camera.DSiCameraSource
 import me.magnum.melonds.common.romprocessors.RomFileProcessorFactory
-import me.magnum.melonds.common.runtime.FrameBufferProvider
+import me.magnum.melonds.common.runtime.ScreenshotFrameBufferProvider
 import me.magnum.melonds.common.uridelegates.UriHandler
 import me.magnum.melonds.domain.model.camera.DSiCameraSourceType
 import me.magnum.melonds.domain.repositories.SettingsRepository
@@ -40,8 +40,8 @@ object EmulatorRuntimeModule {
 
     @Provides
     @ActivityRetainedScoped
-    fun provideFrameBufferProvider(): FrameBufferProvider {
-        return FrameBufferProvider()
+    fun provideFrameBufferProvider(): ScreenshotFrameBufferProvider {
+        return ScreenshotFrameBufferProvider()
     }
 
     @Provides
@@ -113,7 +113,7 @@ object EmulatorRuntimeModule {
         @ApplicationContext context: Context,
         settingsRepository: SettingsRepository,
         sramProvider: SramProvider,
-        frameBufferProvider: FrameBufferProvider,
+        screenshotFrameBufferProvider: ScreenshotFrameBufferProvider,
         romFileProcessorFactory: RomFileProcessorFactory,
         permissionHandler: PermissionHandler,
         cameraManagerMultiplexer: DSiCameraSourceMultiplexer,
@@ -122,7 +122,7 @@ object EmulatorRuntimeModule {
             context,
             settingsRepository,
             sramProvider,
-            frameBufferProvider,
+            screenshotFrameBufferProvider,
             romFileProcessorFactory,
             permissionHandler,
             cameraManagerMultiplexer,

app/src/main/java/me/magnum/melonds/domain/model/RendererConfiguration.kt

@@ -2,16 +2,11 @@ package me.magnum.melonds.domain.model
 
 data class RendererConfiguration(
     val renderer: VideoRenderer,
-    private val internalVideoFiltering: VideoFiltering,
+    val videoFiltering: VideoFiltering,
     val threadedRendering: Boolean,
     private val internalResolutionScaling: Int,
 ) {
 
-    val videoFiltering get() = when (renderer) {
-        VideoRenderer.SOFTWARE -> internalVideoFiltering
-        VideoRenderer.OPENGL -> VideoFiltering.NONE
-    }
-
     val resolutionScaling get() = when (renderer) {
         VideoRenderer.SOFTWARE -> 1
         VideoRenderer.OPENGL -> internalResolutionScaling