diff --git a/.circleci/config.yml b/.circleci/config.yml index f0efee9..b60b520 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ jobs: build: working_directory: ~/code docker: - - image: cimg/android:2024.08.1 + - image: cimg/android:2024.11.1-ndk environment: GRADLE_OPTS: -Dorg.gradle.workers.max=1 -Dorg.gradle.daemon=false -Dkotlin.compiler.execution.strategy="in-process" steps: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..fa149aa --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "brotli"] + path = brotli + url = https://github.com/google/brotli.git diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 7384013..5b8a93d 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -56,6 +56,17 @@ android { kotlinOptions.jvmTarget = javaVersion.toString() packaging.resources.excludes += "/META-INF/{AL2.0,LGPL2.1}" lint.informational.add("MissingTranslation") + + sourceSets.getByName("main") { + java.srcDirs("../brotli/java") + java.excludes.add("**/brotli/**/*Test.java") + } + externalNativeBuild { + cmake { + path = file("src/main/cpp/CMakeLists.txt") + version = "3.22.1" + } + } } dependencies { @@ -66,7 +77,7 @@ dependencies { implementation(libs.browser) implementation(libs.core.ktx) implementation(libs.firebase.analytics) - implementation(libs.firebase.crashlytics) + implementation(libs.firebase.crashlytics.ndk) implementation(libs.fragment.ktx) implementation(libs.hiddenapibypass) implementation(libs.material) diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000..1f93931 --- /dev/null +++ b/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.22.1) +project("brotli") +set(BROTLI_DIR "../../../../brotli") +file(GLOB COMMON_SOURCES "${BROTLI_DIR}/c/common/*.c") +file(GLOB ENC_SOURCES "${BROTLI_DIR}/c/enc/*.c") +add_library(${CMAKE_PROJECT_NAME} SHARED + ${COMMON_SOURCES} + ${ENC_SOURCES} + ${BROTLI_DIR}/java/org/brotli/wrapper/enc/encoder_jni.cc +) +target_link_libraries(${CMAKE_PROJECT_NAME} + # List libraries link to the target library + #android + #log +) +include_directories( + ${BROTLI_DIR}/c/include +) diff --git a/app/src/main/java/be/mygod/reactmap/ConfigDialogFragment.kt b/app/src/main/java/be/mygod/reactmap/ConfigDialogFragment.kt index f258812..7bc4b2d 100644 --- a/app/src/main/java/be/mygod/reactmap/ConfigDialogFragment.kt +++ b/app/src/main/java/be/mygod/reactmap/ConfigDialogFragment.kt @@ -74,17 +74,18 @@ class ConfigDialogFragment : AlertDialogFragment { + ReactMapHttpEngine.detectBrotliError(conn)?.let { notifyError(it) } + Result.retry() + } else -> { val error = conn.findErrorStream.bufferedReader().readText() notifyErrors(error) diff --git a/app/src/main/java/be/mygod/reactmap/webkit/BaseReactMapFragment.kt b/app/src/main/java/be/mygod/reactmap/webkit/BaseReactMapFragment.kt index 2ad4064..8908be2 100644 --- a/app/src/main/java/be/mygod/reactmap/webkit/BaseReactMapFragment.kt +++ b/app/src/main/java/be/mygod/reactmap/webkit/BaseReactMapFragment.kt @@ -29,6 +29,7 @@ import androidx.annotation.RequiresApi import androidx.core.net.toUri import androidx.core.os.bundleOf import androidx.fragment.app.Fragment +import androidx.lifecycle.lifecycleScope import be.mygod.reactmap.App.Companion.app import be.mygod.reactmap.BuildConfig import be.mygod.reactmap.R @@ -37,6 +38,7 @@ import be.mygod.reactmap.util.UnblockCentral import be.mygod.reactmap.util.findErrorStream import com.google.android.material.snackbar.Snackbar import com.google.firebase.analytics.FirebaseAnalytics +import kotlinx.coroutines.launch import org.json.JSONArray import org.json.JSONException import org.json.JSONObject @@ -123,6 +125,7 @@ abstract class BaseReactMapFragment : Fragment(), DownloadListener { protected lateinit var web: WebView protected lateinit var glocation: Glocation + private lateinit var postInterceptor: PostInterceptor protected lateinit var hostname: String private var loginText: String? = null @@ -152,6 +155,7 @@ abstract class BaseReactMapFragment : Fragment(), DownloadListener { javaScriptEnabled = true } glocation = Glocation(this, this@BaseReactMapFragment) + postInterceptor = PostInterceptor(this) webChromeClient = object : WebChromeClient() { @Suppress("KotlinConstantConditions") override fun onConsoleMessage(consoleMessage: ConsoleMessage) = consoleMessage.run { @@ -194,11 +198,15 @@ abstract class BaseReactMapFragment : Fragment(), DownloadListener { override fun onPageStarted(view: WebView, url: String, favicon: Bitmap?) { glocation.clear() + postInterceptor.clear() val uri = url.toUri() if (!BuildConfig.DEBUG && "http".equals(uri.scheme, true)) { web.loadUrl(uri.buildUpon().scheme("https").build().toString()) } - if (uri.host == hostname) glocation.setupGeolocation() + if (uri.host == hostname) { + glocation.setupGeolocation() + postInterceptor.setup() + } onPageStarted() } @@ -233,9 +241,7 @@ abstract class BaseReactMapFragment : Fragment(), DownloadListener { return handleTranslation(request) } if (vendorJsMatcher.matchEntire(path) != null) return handleVendorJs(request) - if (path == "/graphql" && request.method == "POST") { - request.requestHeaders.remove("_interceptedBody")?.let { return handleGraphql(request, it) } - } + postInterceptor.extractBody(request)?.let { return handleGraphql(request, it) } } if (ReactMapHttpEngine.isCronet && (path.substringAfterLast('.').lowercase(Locale.ENGLISH) in mediaExtensions || request.requestHeaders.any { (key, value) -> @@ -374,7 +380,12 @@ abstract class BaseReactMapFragment : Fragment(), DownloadListener { setupConnection(request, conn) ReactMapHttpEngine.writeCompressed(conn, body) } - createResponse(conn) { _ -> conn.findErrorStream } + if (conn.responseCode == 302) { + ReactMapHttpEngine.detectBrotliError(conn)?.let { + lifecycleScope.launch { Snackbar.make(web, it, Snackbar.LENGTH_LONG).show() } + } + null + } else createResponse(conn) { _ -> conn.findErrorStream } } catch (e: IOException) { Timber.d(e) null diff --git a/app/src/main/java/be/mygod/reactmap/webkit/Glocation.kt b/app/src/main/java/be/mygod/reactmap/webkit/Glocation.kt index 17cc477..d4c3abd 100644 --- a/app/src/main/java/be/mygod/reactmap/webkit/Glocation.kt +++ b/app/src/main/java/be/mygod/reactmap/webkit/Glocation.kt @@ -53,7 +53,7 @@ class Glocation(private val web: WebView, private val fragment: BaseReactMapFrag fragment.lifecycle.addObserver(this) it.context } - private val jsSetup = fragment.resources.openRawResource(R.raw.setup).bufferedReader().readText() + private val jsSetup = fragment.resources.openRawResource(R.raw.setup_glocation).bufferedReader().readText() private val pendingRequests = mutableSetOf() private var pendingWatch = false private val activeListeners = mutableSetOf() diff --git a/app/src/main/java/be/mygod/reactmap/webkit/PostInterceptor.kt b/app/src/main/java/be/mygod/reactmap/webkit/PostInterceptor.kt new file mode 100644 index 0000000..6c0a9a3 --- /dev/null +++ b/app/src/main/java/be/mygod/reactmap/webkit/PostInterceptor.kt @@ -0,0 +1,32 @@ +package be.mygod.reactmap.webkit + +import android.util.LongSparseArray +import android.webkit.JavascriptInterface +import android.webkit.WebResourceRequest +import android.webkit.WebView +import be.mygod.reactmap.R +import com.google.common.hash.Hashing +import java.nio.charset.Charset + +class PostInterceptor(private val web: WebView) { + private val bodyLookup = LongSparseArray().also { + web.addJavascriptInterface(this, "_postInterceptor") + } + private val jsSetup = web.resources.openRawResource(R.raw.setup_interceptor).bufferedReader().readText() + + fun setup() = web.evaluateJavascript(jsSetup, null) + + @JavascriptInterface + fun register(body: String): String { + val key = Hashing.sipHash24().hashString(body, Charset.defaultCharset()).asLong() + synchronized(bodyLookup) { bodyLookup.put(key, body) } + return key.toULong().toString(36) + } + fun extractBody(request: WebResourceRequest) = request.requestHeaders.remove("Body-Digest")?.let { key -> + synchronized(bodyLookup) { + val index = bodyLookup.indexOfKey(key.toULong(36).toLong()) + if (index < 0) null else bodyLookup.valueAt(index).also { bodyLookup.removeAt(index) } + } + } + fun clear() = synchronized(bodyLookup) { bodyLookup.clear() } +} diff --git a/app/src/main/java/be/mygod/reactmap/webkit/ReactMapHttpEngine.kt b/app/src/main/java/be/mygod/reactmap/webkit/ReactMapHttpEngine.kt index 64e1300..39de4cd 100644 --- a/app/src/main/java/be/mygod/reactmap/webkit/ReactMapHttpEngine.kt +++ b/app/src/main/java/be/mygod/reactmap/webkit/ReactMapHttpEngine.kt @@ -1,5 +1,6 @@ package be.mygod.reactmap.webkit +import android.net.Uri import android.net.http.ConnectionMigrationOptions import android.net.http.HttpEngine import android.os.Build @@ -14,6 +15,9 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.launch import kotlinx.coroutines.suspendCancellableCoroutine +import org.brotli.wrapper.enc.BrotliOutputStream +import org.brotli.wrapper.enc.Encoder +import timber.log.Timber import java.io.ByteArrayOutputStream import java.io.File import java.net.HttpURLConnection @@ -25,6 +29,7 @@ import kotlin.coroutines.resumeWithException object ReactMapHttpEngine { private const val KEY_COOKIE = "cookie.graphql" + const val KEY_BROTLI = "http.brotli" val isCronet get() = Build.VERSION.SDK_INT >= 34 || Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && SdkExtensions.getExtensionVersion(Build.VERSION_CODES.S) >= 7 @@ -44,9 +49,10 @@ object ReactMapHttpEngine { }.build() } - val apiUrl get() = app.activeUrl.toUri().buildUpon().apply { + fun apiUrl(base: Uri) = base.buildUpon().apply { path("/graphql") }.build().toString() + val apiUrl get() = apiUrl(app.activeUrl.toUri()) private fun openConnection(url: String) = (if (isCronet) { engine.openConnection(URL(url)) @@ -92,16 +98,33 @@ object ReactMapHttpEngine { val buffer get() = buf val length get() = count } + private val initBrotli by lazy { System.loadLibrary("brotli") } fun writeCompressed(conn: HttpURLConnection, body: String) { - conn.setRequestProperty("Content-Encoding", "deflate") + val brotli = app.pref.getBoolean(KEY_BROTLI, true) + conn.setRequestProperty("Content-Encoding", if (brotli) { + initBrotli + "br" + } else "deflate") conn.doOutput = true + conn.instanceFollowRedirects = false val uncompressed = body.toByteArray() val out = ExposingBufferByteArrayOutputStream() - DeflaterOutputStream(out, Deflater(Deflater.BEST_COMPRESSION)).use { - it.write(uncompressed) - } - // Timber.tag("CompressionStat").i("${out.length}/${uncompressed.size} ~ ${out.length.toDouble() / uncompressed.size}") +// val time = System.nanoTime() + (if (brotli) BrotliOutputStream(out, Encoder.Parameters().apply { + setMode(Encoder.Mode.TEXT) + setQuality(5) + }) else DeflaterOutputStream(out, Deflater(Deflater.BEST_COMPRESSION))).use { it.write(uncompressed) } +// Timber.tag("CompressionStat").i("$brotli ${out.length}/${uncompressed.size} ~ ${out.length.toDouble() / uncompressed.size} ${(System.nanoTime() - time) * .000_001}ms") conn.setFixedLengthStreamingMode(out.length) conn.outputStream.use { it.write(out.buffer, 0, out.length) } } + + fun detectBrotliError(conn: HttpURLConnection): String? { + val path = conn.getHeaderField("Location") + if (path.startsWith("/error/")) return Uri.decode(path.substring(7)).also { + if (conn.url.host == app.activeUrl.toUri().host && it == "unsupported content encoding \"br\"") app.pref.edit { putBoolean(KEY_BROTLI, false) } + } + Timber.w(Exception(path)) + return path + } } diff --git a/app/src/main/res/raw/setup.js b/app/src/main/res/raw/setup_glocation.js similarity index 88% rename from app/src/main/res/raw/setup.js rename to app/src/main/res/raw/setup_glocation.js index dad1894..c23fc85 100644 --- a/app/src/main/res/raw/setup.js +++ b/app/src/main/res/raw/setup_glocation.js @@ -48,10 +48,3 @@ Object.defineProperty(navigator, 'geolocation', { }, }, }); -window._fetch = window.fetch; -window.fetch = function (input, init = {}) { - if (input === '/graphql' && init.method === 'POST' && init.body) { - init.headers['_interceptedBody'] = init.body; - } - return window._fetch(input, init); -}; diff --git a/app/src/main/res/raw/setup_interceptor.js b/app/src/main/res/raw/setup_interceptor.js new file mode 100644 index 0000000..647f997 --- /dev/null +++ b/app/src/main/res/raw/setup_interceptor.js @@ -0,0 +1,7 @@ +window._fetch = window.fetch; +window.fetch = function (input, init = {}) { + if (input === '/graphql' && init.method === 'POST' && init.body) { + init.headers['Body-Digest'] = window._postInterceptor.register(init.body); + } + return window._fetch(input, init); +}; diff --git a/brotli b/brotli new file mode 160000 index 0000000..ed738e8 --- /dev/null +++ b/brotli @@ -0,0 +1 @@ +Subproject commit ed738e842d2fbdf2d6459e39267a633c4a9b2f5d diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 8b325d1..fad0f86 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -7,7 +7,7 @@ desugar = { group = "com.android.tools", name = "desugar_jdk_libs", version = "2 espresso-core = { group = "androidx.test.espresso", name = "espresso-core", version = "3.6.1" } firebase-analytics = { group = "com.google.firebase", name = "firebase-analytics" } firebase-bom = { group = "com.google.firebase", name = "firebase-bom", version = "33.5.1" } -firebase-crashlytics = { group = "com.google.firebase", name = "firebase-crashlytics" } +firebase-crashlytics-ndk = { group = "com.google.firebase", name = "firebase-crashlytics-ndk" } fragment-ktx = { group = "androidx.fragment", name = "fragment-ktx", version = "1.8.5" } hiddenapibypass = { group = "org.lsposed.hiddenapibypass", name = "hiddenapibypass", version = "4.3" } junit = { group = "junit", name = "junit", version = "4.13.2" }