Add support for non-HTTPs feeds (#808)

fixes #493
msasikanth · Feb 23, 2025 · 3b1cf81 · 3b1cf81
1 parent f2e814a
commit 3b1cf81
Show file tree

Hide file tree

Showing 9 changed files with 73 additions and 66 deletions.
diff --git a/androidApp/src/androidMain/AndroidManifest.xml b/androidApp/src/androidMain/AndroidManifest.xml
@@ -22,6 +22,8 @@
         android:fullBackupContent="false"
         android:dataExtractionRules="@xml/data_extraction_rules"
         android:enableOnBackInvokedCallback="true"
+        android:usesCleartextTraffic="true"
+        android:networkSecurityConfig="@xml/network_security_config"
         tools:targetApi="tiramisu"
         android:largeHeap="true">
 

diff --git a/androidApp/src/androidMain/res/xml/network_security_config.xml b/androidApp/src/androidMain/res/xml/network_security_config.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?><!--
+  ~ Copyright 2025 Sasikanth Miriyampalli
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<network-security-config xmlns:tools="http://schemas.android.com/tools">
+    <base-config cleartextTrafficPermitted="true"
+        tools:ignore="InsecureBaseConfiguration">
+        <trust-anchors>
+            <certificates src="system" />
+        </trust-anchors>
+    </base-config>
+</network-security-config>
diff --git a/core/data/src/commonMain/kotlin/dev/sasikanth/rss/reader/data/repository/RssRepository.kt b/core/data/src/commonMain/kotlin/dev/sasikanth/rss/reader/data/repository/RssRepository.kt
@@ -74,10 +74,9 @@ class RssRepository(
     feedLink: String,
     title: String? = null,
     feedLastCleanUpAt: Instant? = null,
-    transformUrl: Boolean = true,
   ): FeedAddResult {
     return withContext(ioDispatcher) {
-      when (val feedFetchResult = feedFetcher.fetch(url = feedLink, transformUrl = transformUrl)) {
+      when (val feedFetchResult = feedFetcher.fetch(url = feedLink)) {
         is FeedFetchResult.Success -> {
           return@withContext try {
             val feedPayload = feedFetchResult.feedPayload
@@ -140,13 +139,7 @@ class RssRepository(
         val feedsChunk = feedQueries.feeds().executeAsList().chunked(UPDATE_CHUNKS)
         feedsChunk.map { feeds ->
           feeds.map { feed ->
-            launch {
-              addFeed(
-                feedLink = feed.link,
-                transformUrl = false,
-                feedLastCleanUpAt = feed.lastCleanUpAt
-              )
-            }
+            launch { addFeed(feedLink = feed.link, feedLastCleanUpAt = feed.lastCleanUpAt) }
           }
         }
       }
@@ -158,7 +151,7 @@ class RssRepository(
     withContext(ioDispatcher) {
       val feed = feedQueries.feed(selectedFeedId).executeAsOneOrNull()
       if (feed != null) {
-        addFeed(feedLink = feed.link, transformUrl = false, feedLastCleanUpAt = feed.lastCleanUpAt)
+        addFeed(feedLink = feed.link, feedLastCleanUpAt = feed.lastCleanUpAt)
       }
     }
   }
@@ -168,11 +161,7 @@ class RssRepository(
       feedIds.forEach { feedId ->
         val feed = feedQueries.feed(feedId).executeAsOneOrNull()
         if (feed != null) {
-          addFeed(
-            feedLink = feed.link,
-            transformUrl = false,
-            feedLastCleanUpAt = feed.lastCleanUpAt
-          )
+          addFeed(feedLink = feed.link, feedLastCleanUpAt = feed.lastCleanUpAt)
         }
       }
     }

diff --git a/...twork/src/androidMain/kotlin/dev/sasikanth/rss/reader/core/network/di/NetworkComponent.kt b/...twork/src/androidMain/kotlin/dev/sasikanth/rss/reader/core/network/di/NetworkComponent.kt
@@ -20,12 +20,22 @@ import dev.sasikanth.rss.reader.di.scopes.AppScope
 import io.ktor.client.HttpClient
 import io.ktor.client.engine.okhttp.OkHttp
 import me.tatarka.inject.annotations.Provides
+import okhttp3.Protocol
 
 actual interface NetworkComponent {
 
   @Provides
   @AppScope
   fun providesHttpClient(): HttpClient {
-    return httpClient(engine = OkHttp, config = { config { retryOnConnectionFailure(true) } })
+    return httpClient(
+      engine = OkHttp,
+      config = {
+        config {
+          retryOnConnectionFailure(true)
+
+          protocols(listOf(Protocol.HTTP_1_1, Protocol.HTTP_2))
+        }
+      }
+    )
   }
 }
diff --git a/...etwork/src/commonMain/kotlin/dev/sasikanth/rss/reader/core/network/fetcher/FeedFetcher.kt b/...etwork/src/commonMain/kotlin/dev/sasikanth/rss/reader/core/network/fetcher/FeedFetcher.kt
@@ -31,9 +31,9 @@ import io.ktor.client.statement.bodyAsText
 import io.ktor.http.ContentType
 import io.ktor.http.HttpStatusCode
 import io.ktor.http.URLBuilder
-import io.ktor.http.URLProtocol
 import io.ktor.http.Url
 import io.ktor.http.contentType
+import io.ktor.http.isRelativePath
 import korlibs.io.lang.Charset
 import korlibs.io.lang.Charsets
 import me.tatarka.inject.annotations.Inject
@@ -45,23 +45,20 @@ class FeedFetcher(private val httpClient: HttpClient, private val feedParser: Fe
     private const val MAX_REDIRECTS_ALLOWED = 5
   }
 
-  suspend fun fetch(url: String, transformUrl: Boolean = true): FeedFetchResult {
-    return fetch(url, transformUrl, redirectCount = 0)
+  suspend fun fetch(url: String): FeedFetchResult {
+    return fetch(url, redirectCount = 0)
   }
 
   private suspend fun fetch(
     url: String,
-    transformUrl: Boolean,
     redirectCount: Int,
   ): FeedFetchResult {
     if (redirectCount >= MAX_REDIRECTS_ALLOWED) {
       return FeedFetchResult.TooManyRedirects
     }
 
     return try {
-      // We are mainly doing this to avoid creating duplicates while refreshing feeds
-      // after the app update
-      val transformedUrl = transformUrl(url, transformUrl)
+      val transformedUrl = buildFeedUrl(url)
       val response = httpClient.get(transformedUrl.toString())
 
       when (response.status) {
@@ -85,6 +82,21 @@ class FeedFetcher(private val httpClient: HttpClient, private val feedParser: Fe
     }
   }
 
+  private fun buildFeedUrl(urlString: String): Url {
+    val url = Url(urlString)
+
+    return if (url.isRelativePath) {
+      URLBuilder()
+        .apply {
+          host = urlString
+          protocol = url.protocol
+        }
+        .build()
+    } else {
+      url
+    }
+  }
+
   private suspend fun parseContent(
     response: HttpResponse,
     url: String,
@@ -103,7 +115,7 @@ class FeedFetcher(private val httpClient: HttpClient, private val feedParser: Fe
     val feedUrl = fetchFeedLinkFromHtmlIfExists(response.bodyAsText(), url)
 
     if (feedUrl != url && !feedUrl.isNullOrBlank()) {
-      return fetch(url = feedUrl, transformUrl = false, redirectCount = redirectCount + 1)
+      return fetch(url = feedUrl, redirectCount = redirectCount + 1)
     }
 
     throw UnsupportedOperationException()
@@ -121,27 +133,7 @@ class FeedFetcher(private val httpClient: HttpClient, private val feedParser: Fe
       return FeedFetchResult.Error(Exception("Failed to fetch the feed"))
     }
 
-    return fetch(url = redirectToUrl, transformUrl = true, redirectCount = redirectCount + 1)
-  }
-
-  private fun transformUrl(url: String, transformUrl: Boolean): Url {
-    return if (transformUrl) {
-      // Currently Ktor Url parses relative URLs,
-      // if it fails to properly parse the given URL, it
-      // default to localhost.
-      //
-      // This will cause the network call to fail,
-      // so we are setting the host manually
-      // https://youtrack.jetbrains.com/issue/KTOR-360
-      URLBuilder()
-        .apply {
-          protocol = URLProtocol.HTTPS
-          host = url.replace(Regex("^https?://"), "")
-        }
-        .build()
-    } else {
-      URLBuilder(url).apply { protocol = URLProtocol.HTTPS }.build()
-    }
+    return fetch(url = redirectToUrl, redirectCount = redirectCount + 1)
   }
 
   private fun fetchFeedLinkFromHtmlIfExists(htmlContent: String, originalUrl: String): String? {
@@ -159,9 +151,8 @@ class FeedFetcher(private val httpClient: HttpClient, private val feedParser: Fe
       }
         ?: return null
     val link = linkElement.attr(ATTR_HREF)
-    val host = URLBuilder(originalUrl).build().host
-    val rootUrl = "https://$host"
+    val host = UrlUtils.extractHost(originalUrl)
 
-    return UrlUtils.safeUrl(rootUrl, link)
+    return UrlUtils.safeUrl(host, link)
   }
 }
diff --git a/...rk/src/commonMain/kotlin/dev/sasikanth/rss/reader/core/network/post/FullArticleFetcher.kt b/...rk/src/commonMain/kotlin/dev/sasikanth/rss/reader/core/network/post/FullArticleFetcher.kt
@@ -23,9 +23,6 @@ import io.ktor.client.request.get
 import io.ktor.client.statement.bodyAsText
 import io.ktor.http.ContentType
 import io.ktor.http.HttpStatusCode
-import io.ktor.http.URLBuilder
-import io.ktor.http.URLProtocol
-import io.ktor.http.Url
 import io.ktor.http.contentType
 import kotlinx.coroutines.withContext
 import me.tatarka.inject.annotations.Inject
@@ -40,7 +37,7 @@ class FullArticleFetcher(
   suspend fun fetch(link: String): Result<String> {
     return withContext(dispatchersProvider.io) {
       try {
-        val response = httpClient.get(transformUrlToHttps(link))
+        val response = httpClient.get(link)
         if (
           response.status == HttpStatusCode.OK &&
             response.contentType()?.withoutParameters() == ContentType.Text.Html
@@ -55,8 +52,4 @@ class FullArticleFetcher(
       return@withContext Result.failure(IllegalArgumentException("Failed to fetch the post"))
     }
   }
-
-  private fun transformUrlToHttps(url: String): Url {
-    return URLBuilder(url).apply { protocol = URLProtocol.HTTPS }.build()
-  }
 }
diff --git a/core/network/src/commonMain/kotlin/dev/sasikanth/rss/reader/core/network/utils/UrlUtils.kt b/core/network/src/commonMain/kotlin/dev/sasikanth/rss/reader/core/network/utils/UrlUtils.kt
@@ -17,7 +17,6 @@
 package dev.sasikanth.rss.reader.core.network.utils
 
 import io.ktor.http.URLBuilder
-import io.ktor.http.URLProtocol
 import io.ktor.http.Url
 import io.ktor.http.set
 
@@ -43,14 +42,9 @@ object UrlUtils {
 
     return if (!url.isNullOrBlank()) {
       if (isAbsoluteUrl(url)) {
-        URLBuilder(url).apply { protocol = URLProtocol.HTTPS }.buildString()
+        URLBuilder(url).buildString()
       } else {
-        URLBuilder()
-          .apply {
-            set(host = host, path = url)
-            protocol = URLProtocol.HTTPS
-          }
-          .buildString()
+        URLBuilder().apply { set(host = host, path = url) }.buildString()
       }
     } else {
       null

diff --git a/iosApp/iosApp/Info.plist b/iosApp/iosApp/Info.plist
@@ -69,5 +69,10 @@
 	</array>
 	<key>UIUserInterfaceStyle</key>
 	<string>Automatic</string>
+    <key>NSAppTransportSecurity</key>
+    <dict>
+        <key>NSAllowsArbitraryLoads</key>
+        <true/>
+    </dict>
 </dict>
 </plist>
diff --git a/shared/src/commonTest/kotlin/dev/sasikanth/rss/reader/FeedParserTest.kt b/shared/src/commonTest/kotlin/dev/sasikanth/rss/reader/FeedParserTest.kt
@@ -96,7 +96,7 @@ class FeedParserTest {
               link = "https://example.com/post-with-relative-image",
               description = "Relative image post description.",
               rawContent = "Relative image post description.",
-              imageUrl = "https://example.com/relative-media-url",
+              imageUrl = "http://example.com/relative-media-url",
               date = 1685005200000,
               commentsLink = null
             ),
@@ -228,7 +228,7 @@ class FeedParserTest {
                   <p>Post summary with an image.</p>
                 """
                   .trimIndent(),
-              imageUrl = "https://example.com/resources/image.jpg",
+              imageUrl = "http://example.com/resources/image.jpg",
               date = 1685008800000,
               commentsLink = null
             ),