Skip to content

Support brotli compression #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Mygod merged 1 commit into from
Nov 14, 2024
Merged

Support brotli compression #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ jobs:
build:
working_directory: ~/code
docker:
- image: cimg/android:2024.08.1
- image: cimg/android:2024.11.1-ndk
environment:
GRADLE_OPTS: -Dorg.gradle.workers.max=1 -Dorg.gradle.daemon=false -Dkotlin.compiler.execution.strategy="in-process"
steps:
Expand Down
3 changes: 3 additions & 0 deletions .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[submodule "brotli"]
path = brotli
url = https://github.com/google/brotli.git
13 changes: 12 additions & 1 deletion app/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,17 @@ android {
kotlinOptions.jvmTarget = javaVersion.toString()
packaging.resources.excludes += "/META-INF/{AL2.0,LGPL2.1}"
lint.informational.add("MissingTranslation")

sourceSets.getByName("main") {
java.srcDirs("../brotli/java")
java.excludes.add("**/brotli/**/*Test.java")
}
externalNativeBuild {
cmake {
path = file("src/main/cpp/CMakeLists.txt")
version = "3.22.1"
}
}
}

dependencies {
Expand All @@ -66,7 +77,7 @@ dependencies {
implementation(libs.browser)
implementation(libs.core.ktx)
implementation(libs.firebase.analytics)
implementation(libs.firebase.crashlytics)
implementation(libs.firebase.crashlytics.ndk)
implementation(libs.fragment.ktx)
implementation(libs.hiddenapibypass)
implementation(libs.material)
Expand Down
18 changes: 18 additions & 0 deletions app/src/main/cpp/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
cmake_minimum_required(VERSION 3.22.1)
project("brotli")
set(BROTLI_DIR "../../../../brotli")
file(GLOB COMMON_SOURCES "${BROTLI_DIR}/c/common/*.c")
file(GLOB ENC_SOURCES "${BROTLI_DIR}/c/enc/*.c")
add_library(${CMAKE_PROJECT_NAME} SHARED
${COMMON_SOURCES}
${ENC_SOURCES}
${BROTLI_DIR}/java/org/brotli/wrapper/enc/encoder_jni.cc
)
target_link_libraries(${CMAKE_PROJECT_NAME}
# List libraries link to the target library
#android
#log
)
include_directories(
${BROTLI_DIR}/c/include
)
17 changes: 9 additions & 8 deletions app/src/main/java/be/mygod/reactmap/ConfigDialogFragment.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,17 +74,18 @@ class ConfigDialogFragment : AlertDialogFragment<ConfigDialogFragment.Arg, Empty
}

override val ret get() = try {
val uri = urlEdit.text!!.toString().toUri().let {
require(BuildConfig.DEBUG || "https".equals(it.scheme, true)) { getText(R.string.error_https_only) }
it.host!!
it.toString()
}
val uri = urlEdit.text!!.toString().toUri()
require(BuildConfig.DEBUG || "https".equals(uri.scheme, true)) { getText(R.string.error_https_only) }
uri.host!!
val uriString = uri.toString()
val oldApiUrl = ReactMapHttpEngine.apiUrl
val changing = oldApiUrl != ReactMapHttpEngine.apiUrl(uri)
app.pref.edit {
putString(App.KEY_ACTIVE_URL, uri)
putStringSet(KEY_HISTORY_URL, historyUrl + uri)
putString(App.KEY_ACTIVE_URL, uriString)
putStringSet(KEY_HISTORY_URL, historyUrl + uriString)
if (changing) remove(ReactMapHttpEngine.KEY_BROTLI)
}
if (oldApiUrl != ReactMapHttpEngine.apiUrl) BackgroundLocationReceiver.onApiChanged()
if (changing) BackgroundLocationReceiver.onApiChanged()
Empty()
} catch (e: Exception) {
Toast.makeText(requireContext(), e.readableMessage, Toast.LENGTH_LONG).show()
Expand Down
4 changes: 4 additions & 0 deletions app/src/main/java/be/mygod/reactmap/follower/LocationSetter.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,10 @@ class LocationSetter(appContext: Context, workerParams: WorkerParameters) : Coro
}.build())
Result.success()
}
302 -> {
ReactMapHttpEngine.detectBrotliError(conn)?.let { notifyError(it) }
Result.retry()
}
else -> {
val error = conn.findErrorStream.bufferedReader().readText()
notifyErrors(error)
Expand Down
21 changes: 16 additions & 5 deletions app/src/main/java/be/mygod/reactmap/webkit/BaseReactMapFragment.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import androidx.annotation.RequiresApi
import androidx.core.net.toUri
import androidx.core.os.bundleOf
import androidx.fragment.app.Fragment
import androidx.lifecycle.lifecycleScope
import be.mygod.reactmap.App.Companion.app
import be.mygod.reactmap.BuildConfig
import be.mygod.reactmap.R
Expand All @@ -37,6 +38,7 @@ import be.mygod.reactmap.util.UnblockCentral
import be.mygod.reactmap.util.findErrorStream
import com.google.android.material.snackbar.Snackbar
import com.google.firebase.analytics.FirebaseAnalytics
import kotlinx.coroutines.launch
import org.json.JSONArray
import org.json.JSONException
import org.json.JSONObject
Expand Down Expand Up @@ -123,6 +125,7 @@ abstract class BaseReactMapFragment : Fragment(), DownloadListener {

protected lateinit var web: WebView
protected lateinit var glocation: Glocation
private lateinit var postInterceptor: PostInterceptor
protected lateinit var hostname: String

private var loginText: String? = null
Expand Down Expand Up @@ -152,6 +155,7 @@ abstract class BaseReactMapFragment : Fragment(), DownloadListener {
javaScriptEnabled = true
}
glocation = Glocation(this, this@BaseReactMapFragment)
postInterceptor = PostInterceptor(this)
webChromeClient = object : WebChromeClient() {
@Suppress("KotlinConstantConditions")
override fun onConsoleMessage(consoleMessage: ConsoleMessage) = consoleMessage.run {
Expand Down Expand Up @@ -194,11 +198,15 @@ abstract class BaseReactMapFragment : Fragment(), DownloadListener {

override fun onPageStarted(view: WebView, url: String, favicon: Bitmap?) {
glocation.clear()
postInterceptor.clear()
val uri = url.toUri()
if (!BuildConfig.DEBUG && "http".equals(uri.scheme, true)) {
web.loadUrl(uri.buildUpon().scheme("https").build().toString())
}
if (uri.host == hostname) glocation.setupGeolocation()
if (uri.host == hostname) {
glocation.setupGeolocation()
postInterceptor.setup()
}
onPageStarted()
}

Expand Down Expand Up @@ -233,9 +241,7 @@ abstract class BaseReactMapFragment : Fragment(), DownloadListener {
return handleTranslation(request)
}
if (vendorJsMatcher.matchEntire(path) != null) return handleVendorJs(request)
if (path == "/graphql" && request.method == "POST") {
request.requestHeaders.remove("_interceptedBody")?.let { return handleGraphql(request, it) }
}
postInterceptor.extractBody(request)?.let { return handleGraphql(request, it) }
}
if (ReactMapHttpEngine.isCronet && (path.substringAfterLast('.').lowercase(Locale.ENGLISH)
in mediaExtensions || request.requestHeaders.any { (key, value) ->
Expand Down Expand Up @@ -374,7 +380,12 @@ abstract class BaseReactMapFragment : Fragment(), DownloadListener {
setupConnection(request, conn)
ReactMapHttpEngine.writeCompressed(conn, body)
}
createResponse(conn) { _ -> conn.findErrorStream }
if (conn.responseCode == 302) {
ReactMapHttpEngine.detectBrotliError(conn)?.let {
lifecycleScope.launch { Snackbar.make(web, it, Snackbar.LENGTH_LONG).show() }
}
null
} else createResponse(conn) { _ -> conn.findErrorStream }
} catch (e: IOException) {
Timber.d(e)
null
Expand Down
2 changes: 1 addition & 1 deletion app/src/main/java/be/mygod/reactmap/webkit/Glocation.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ class Glocation(private val web: WebView, private val fragment: BaseReactMapFrag
fragment.lifecycle.addObserver(this)
it.context
}
private val jsSetup = fragment.resources.openRawResource(R.raw.setup).bufferedReader().readText()
private val jsSetup = fragment.resources.openRawResource(R.raw.setup_glocation).bufferedReader().readText()
private val pendingRequests = mutableSetOf<Long>()
private var pendingWatch = false
private val activeListeners = mutableSetOf<Long>()
Expand Down
32 changes: 32 additions & 0 deletions app/src/main/java/be/mygod/reactmap/webkit/PostInterceptor.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package be.mygod.reactmap.webkit

import android.util.LongSparseArray
import android.webkit.JavascriptInterface
import android.webkit.WebResourceRequest
import android.webkit.WebView
import be.mygod.reactmap.R
import com.google.common.hash.Hashing
import java.nio.charset.Charset

class PostInterceptor(private val web: WebView) {
private val bodyLookup = LongSparseArray<String>().also {
web.addJavascriptInterface(this, "_postInterceptor")
}
private val jsSetup = web.resources.openRawResource(R.raw.setup_interceptor).bufferedReader().readText()

fun setup() = web.evaluateJavascript(jsSetup, null)

@JavascriptInterface
fun register(body: String): String {
val key = Hashing.sipHash24().hashString(body, Charset.defaultCharset()).asLong()
synchronized(bodyLookup) { bodyLookup.put(key, body) }
return key.toULong().toString(36)
}
fun extractBody(request: WebResourceRequest) = request.requestHeaders.remove("Body-Digest")?.let { key ->
synchronized(bodyLookup) {
val index = bodyLookup.indexOfKey(key.toULong(36).toLong())
if (index < 0) null else bodyLookup.valueAt(index).also { bodyLookup.removeAt(index) }
}
}
fun clear() = synchronized(bodyLookup) { bodyLookup.clear() }
}
35 changes: 29 additions & 6 deletions app/src/main/java/be/mygod/reactmap/webkit/ReactMapHttpEngine.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package be.mygod.reactmap.webkit

import android.net.Uri
import android.net.http.ConnectionMigrationOptions
import android.net.http.HttpEngine
import android.os.Build
Expand All @@ -14,6 +15,9 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.launch
import kotlinx.coroutines.suspendCancellableCoroutine
import org.brotli.wrapper.enc.BrotliOutputStream
import org.brotli.wrapper.enc.Encoder
import timber.log.Timber
import java.io.ByteArrayOutputStream
import java.io.File
import java.net.HttpURLConnection
Expand All @@ -25,6 +29,7 @@ import kotlin.coroutines.resumeWithException

object ReactMapHttpEngine {
private const val KEY_COOKIE = "cookie.graphql"
const val KEY_BROTLI = "http.brotli"

val isCronet get() = Build.VERSION.SDK_INT >= 34 || Build.VERSION.SDK_INT >= Build.VERSION_CODES.S &&
SdkExtensions.getExtensionVersion(Build.VERSION_CODES.S) >= 7
Expand All @@ -44,9 +49,10 @@ object ReactMapHttpEngine {
}.build()
}

val apiUrl get() = app.activeUrl.toUri().buildUpon().apply {
fun apiUrl(base: Uri) = base.buildUpon().apply {
path("/graphql")
}.build().toString()
val apiUrl get() = apiUrl(app.activeUrl.toUri())

private fun openConnection(url: String) = (if (isCronet) {
engine.openConnection(URL(url))
Expand Down Expand Up @@ -92,16 +98,33 @@ object ReactMapHttpEngine {
val buffer get() = buf
val length get() = count
}
private val initBrotli by lazy { System.loadLibrary("brotli") }
fun writeCompressed(conn: HttpURLConnection, body: String) {
conn.setRequestProperty("Content-Encoding", "deflate")
val brotli = app.pref.getBoolean(KEY_BROTLI, true)
conn.setRequestProperty("Content-Encoding", if (brotli) {
initBrotli
"br"
} else "deflate")
conn.doOutput = true
conn.instanceFollowRedirects = false
val uncompressed = body.toByteArray()
val out = ExposingBufferByteArrayOutputStream()
DeflaterOutputStream(out, Deflater(Deflater.BEST_COMPRESSION)).use {
it.write(uncompressed)
}
// Timber.tag("CompressionStat").i("${out.length}/${uncompressed.size} ~ ${out.length.toDouble() / uncompressed.size}")
// val time = System.nanoTime()
(if (brotli) BrotliOutputStream(out, Encoder.Parameters().apply {
setMode(Encoder.Mode.TEXT)
setQuality(5)
}) else DeflaterOutputStream(out, Deflater(Deflater.BEST_COMPRESSION))).use { it.write(uncompressed) }
// Timber.tag("CompressionStat").i("$brotli ${out.length}/${uncompressed.size} ~ ${out.length.toDouble() / uncompressed.size} ${(System.nanoTime() - time) * .000_001}ms")
conn.setFixedLengthStreamingMode(out.length)
conn.outputStream.use { it.write(out.buffer, 0, out.length) }
}

fun detectBrotliError(conn: HttpURLConnection): String? {
val path = conn.getHeaderField("Location")
if (path.startsWith("/error/")) return Uri.decode(path.substring(7)).also {
if (conn.url.host == app.activeUrl.toUri().host && it == "unsupported content encoding \"br\"") app.pref.edit { putBoolean(KEY_BROTLI, false) }
}
Timber.w(Exception(path))
return path
}
}
7 changes: 0 additions & 7 deletions app/src/main/res/raw/setup.js → app/src/main/res/raw/setup_glocation.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,3 @@ Object.defineProperty(navigator, 'geolocation', {
},
},
});
window._fetch = window.fetch;
window.fetch = function (input, init = {}) {
if (input === '/graphql' && init.method === 'POST' && init.body) {
init.headers['_interceptedBody'] = init.body;
}
return window._fetch(input, init);
};
7 changes: 7 additions & 0 deletions app/src/main/res/raw/setup_interceptor.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
window._fetch = window.fetch;
window.fetch = function (input, init = {}) {
if (input === '/graphql' && init.method === 'POST' && init.body) {
init.headers['Body-Digest'] = window._postInterceptor.register(init.body);
}
return window._fetch(input, init);
};
1 change: 1 addition & 0 deletions brotli
Open in desktop
Submodule brotli added at ed738e
2 changes: 1 addition & 1 deletion gradle/libs.versions.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ desugar = { group = "com.android.tools", name = "desugar_jdk_libs", version = "2
espresso-core = { group = "androidx.test.espresso", name = "espresso-core", version = "3.6.1" }
firebase-analytics = { group = "com.google.firebase", name = "firebase-analytics" }
firebase-bom = { group = "com.google.firebase", name = "firebase-bom", version = "33.5.1" }
firebase-crashlytics = { group = "com.google.firebase", name = "firebase-crashlytics" }
firebase-crashlytics-ndk = { group = "com.google.firebase", name = "firebase-crashlytics-ndk" }
fragment-ktx = { group = "androidx.fragment", name = "fragment-ktx", version = "1.8.5" }
hiddenapibypass = { group = "org.lsposed.hiddenapibypass", name = "hiddenapibypass", version = "4.3" }
junit = { group = "junit", name = "junit", version = "4.13.2" }
Expand Down