Skip to content

Commit 373edf1

Browse files
crim50ncrim50n
committed
feat: add Qualcomm QNN backend for local diffusion with NPU acceleration
Add new LOCAL_QUALCOMM_QNN server source that enables on-device Stable Diffusion generation using Qualcomm QNN SDK with NPU (HTP) acceleration. Supports Snapdragon 8 Gen 1 and newer chipsets. Features: - NPU-accelerated inference using Qualcomm HTP backend - GPU fallback with OpenCL support (MNN backend) - Built-in models from HuggingFace (xororz/sd-qnn) - Custom model support with folder scanning - Multiple resolution presets (512x512, 768x768, etc.) - Configurable scheduler (DPM, Euler) - Real-time diffusion process visualization - Img2Img support with denoising Technical changes: - Add feature/qnn module with LocalDream-based implementation - Add DeviceChipsetDetector for Snapdragon detection - Add QNN preferences (model, runtime, scheduler, etc.) - Add QnnForm for server setup configuration - Add QnnRuntimeSelectionComponent for runtime switching - Add model scanning for custom QNN models - Update generation usecases to support QNN backend - Add prepare_qnn_libs.sh script for library setup Note: QNN libraries must be extracted separately using the prepare_qnn_libs.sh script (not included in repository).
1 parent 127580e commit 373edf1

97 files changed

Lines changed: 4539 additions & 207 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎app/build.gradle.kts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,13 @@ android {
1515
includeInBundle = false
1616
}
1717

18+
packaging {
19+
jniLibs {
20+
// Required to extract native executable for QNN backend
21+
useLegacyPackaging = true
22+
}
23+
}
24+
1825
defaultConfig {
1926
applicationId = "com.shifthackz.aisdv1.app"
2027
versionName = libs.versions.versionName.get()
@@ -76,6 +83,7 @@ dependencies {
7683
implementation(project(":feature:auth"))
7784
implementation(project(":feature:diffusion"))
7885
implementation(project(":feature:mediapipe"))
86+
implementation(project(":feature:qnn"))
7987
implementation(project(":feature:work"))
8088
implementation(project(":data"))
8189
implementation(project(":demo"))

‎app/src/main/java/com/shifthackz/aisdv1/app/di/FeatureModule.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@ package com.shifthackz.aisdv1.app.di
33
import com.shifthackz.aisdv1.feature.auth.di.authModule
44
import com.shifthackz.aisdv1.feature.diffusion.di.diffusionModule
55
import com.shifthackz.aisdv1.feature.mediapipe.di.mediaPipeModule
6+
import com.shifthackz.aisdv1.feature.qnn.di.qnnModule
67

78
val featureModule = arrayOf(
89
authModule,
910
diffusionModule,
1011
mediaPipeModule,
12+
qnnModule,
1113
)

‎core/common/src/main/java/com/shifthackz/aisdv1/core/common/device/DeviceChipsetDetector.kt‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
package com.shifthackz.aisdv1.core.common.device
2+
3+
import android.os.Build
4+
5+
/**
6+
* Utility for detecting Qualcomm Snapdragon chipset and determining
7+
* appropriate QNN model variant suffix.
8+
*
9+
* QNN models come in three variants:
10+
* - 8gen2: For flagship chips (Snapdragon 8 Gen 2, 3, 4)
11+
* - 8gen1: For older flagship chips (Snapdragon 8 Gen 1)
12+
* - min: For other Snapdragon chips or unknown devices
13+
*/
14+
object DeviceChipsetDetector {
15+
16+
/**
17+
* Chipset model suffixes for QNN model compatibility.
18+
* Maps SOC_MODEL to appropriate model suffix.
19+
*/
20+
private val chipsetModelSuffixes = mapOf(
21+
// Snapdragon 8 Gen 1 family
22+
"SM8450" to "8gen1", // Snapdragon 8 Gen 1
23+
"SM8475" to "8gen1", // Snapdragon 8+ Gen 1
24+
25+
// Snapdragon 8 Gen 2+ family (all use 8gen2 suffix)
26+
"SM8550" to "8gen2", // Snapdragon 8 Gen 2
27+
"SM8550P" to "8gen2",
28+
"QCS8550" to "8gen2",
29+
"QCM8550" to "8gen2",
30+
"SM8650" to "8gen2", // Snapdragon 8 Gen 3
31+
"SM8650P" to "8gen2",
32+
"SM8750" to "8gen2", // Snapdragon 8 Gen 4
33+
"SM8750P" to "8gen2",
34+
"SM8850" to "8gen2", // Future chips
35+
"SM8850P" to "8gen2",
36+
"SM8735" to "8gen2",
37+
"SM8845" to "8gen2",
38+
)
39+
40+
/**
41+
* Get the device SOC model name.
42+
* Returns empty string on older Android versions (< S).
43+
*/
44+
fun getDeviceSoc(): String {
45+
return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
46+
Build.SOC_MODEL
47+
} else {
48+
""
49+
}
50+
}
51+
52+
/**
53+
* Get the chipset suffix for QNN model selection.
54+
*
55+
* @param soc The SOC model string (usually from Build.SOC_MODEL)
56+
* @return One of: "8gen2", "8gen1", or "min"
57+
*/
58+
fun getChipsetSuffix(soc: String = getDeviceSoc()): String {
59+
// Direct match in our known chipsets
60+
chipsetModelSuffixes[soc]?.let { return it }
61+
62+
// For unknown SM* chips, default to "min" for safety
63+
if (soc.startsWith("SM")) {
64+
return "min"
65+
}
66+
67+
// For completely unknown devices, use min
68+
return "min"
69+
}
70+
71+
/**
72+
* Check if the current device has a known Qualcomm Snapdragon chipset.
73+
*/
74+
fun isQualcommDevice(): Boolean {
75+
val soc = getDeviceSoc()
76+
return soc.startsWith("SM") || soc.startsWith("QCS") || soc.startsWith("QCM")
77+
}
78+
79+
/**
80+
* Check if the current device supports 8gen2 QNN models.
81+
*/
82+
fun supports8Gen2(): Boolean = getChipsetSuffix() == "8gen2"
83+
84+
/**
85+
* Check if the current device supports 8gen1 QNN models.
86+
*/
87+
fun supports8Gen1(): Boolean = getChipsetSuffix() == "8gen1"
88+
89+
/**
90+
* Get recommended QNN model suffix for this device.
91+
*/
92+
fun getRecommendedModelSuffix(): String = getChipsetSuffix()
93+
94+
/**
95+
* Get display name for the chipset suffix.
96+
*/
97+
fun getChipsetDisplayName(suffix: String = getChipsetSuffix()): String {
98+
return when (suffix) {
99+
"8gen2" -> "Snapdragon 8 Gen 2/3/4"
100+
"8gen1" -> "Snapdragon 8 Gen 1/+"
101+
"min" -> "Other Snapdragon"
102+
else -> "Unknown"
103+
}
104+
}
105+
}

‎core/common/src/main/java/com/shifthackz/aisdv1/core/common/file/FileExtensions.kt‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ fun File.unzip() {
4747
val destinationDir = parentFile ?: return
4848

4949
fun extractFile(inputStream: InputStream, destFilePath: String) {
50+
val destFile = File(destFilePath)
51+
// Create parent directories if they don't exist
52+
destFile.parentFile?.mkdirs()
5053
val bos = BufferedOutputStream(FileOutputStream(destFilePath))
5154
val bytesIn = ByteArray(DEFAULT_BUFFER_SIZE)
5255
var read: Int
@@ -64,7 +67,7 @@ fun File.unzip() {
6467
extractFile(inputStream, filePath)
6568
} else {
6669
val dir = File(filePath)
67-
dir.mkdir()
70+
dir.mkdirs()
6871
}
6972
}
7073
}

‎core/common/src/main/java/com/shifthackz/aisdv1/core/common/model/Heptagonal.kt‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package com.shifthackz.aisdv1.core.common.model
2+
3+
import java.io.Serializable
4+
5+
data class Heptagonal<out A, out B, out C, out D, out E, out F, out G>(
6+
val first: A,
7+
val second: B,
8+
val third: C,
9+
val fourth: D,
10+
val fifth: E,
11+
val sixth: F,
12+
val seventh: G,
13+
) : Serializable {
14+
15+
override fun toString(): String = "($first, $second, $third, $fourth, $fifth, $sixth, $seventh)"
16+
}

‎core/localization/src/main/res/values-ru/strings.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,8 @@
161161

162162
<string name="hint_mediapipe_sub_title">Эта конфигурация использует Google AI MediaPipe и позволяет запускать генерации Stable Diffusion на вашем телефоне без необходимости подключаться к удаленному серверу/облаку.</string>
163163

164+
<string name="hint_qnn_sub_title">Эта конфигурация использует Qualcomm QNN SDK с ускорением NPU (HTP) и MNN backend для быстрой генерации Stable Diffusion на устройстве. Требуется Snapdragon 8 Gen 1 или новее.</string>
165+
164166
<string name="drawer_web_ui">Веб</string>
165167

166168
<string name="home_tab_txt_to_img">Txt2Img</string>
@@ -310,6 +312,7 @@
310312
<string name="error_localhost_url">Вы не можете использовать localhost (127.0.0.1) URL-адрес для подключения к серверу.</string>
311313
<string name="error_min_size">Минимальный размер %1$s</string>
312314
<string name="error_max_size">Максимальный размер %1$s</string>
315+
<string name="error_qnn_unsupported_resolution">Неподдерживаемое разрешение для QNN. Используйте выпадающий список для выбора корректного разрешения.</string>
313316
<string name="error_download_fail">Ошибка при загрузке</string>
314317
<string name="error_generic">При обработке вашего запроса произошла ошибка. Повторите попытку позже.</string>
315318

@@ -336,6 +339,10 @@
336339

337340
<string name="model_local_custom_title">Чтобы использовать локальную пользовательскую модель, поместите ее в локальную папку в памяти телефона.</string>
338341
<string name="model_local_custom_sub_title">Окончательная структура папок должна быть такой:</string>
342+
<string name="model_qnn_custom_sub_title">Требуемая структура папки QNN:</string>
343+
<string name="action_scan_models">Сканировать модели</string>
344+
<string name="model_scan_empty">Модели не найдены в указанной папке.\n\nУбедитесь, что каждая модель находится в отдельной подпапке с необходимыми файлами.</string>
345+
<string name="model_scan_found">Найдено %1$d моделей:</string>
339346

340347
<string name="debug_section_main">Отладка</string>
341348
<string name="debug_section_qa">QA тест-кейсы</string>

‎core/localization/src/main/res/values-tr/strings.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,8 @@
159159

160160
<string name="hint_mediapipe_sub_title">Bu yapılandırma Google AI MediaPipe çalışma zamanını kullanır ve uzak bir sunucuya/buluta bağlanmaya gerek kalmadan telefonunuzda Stable Diffusion AI nesillerini çalıştırmanıza olanak tanır.</string>
161161

162+
<string name="hint_qnn_sub_title">Bu yapılandırma, cihaz üzerinde hızlı Stable Diffusion üretimi için NPU hızlandırması (HTP) ve MNN arka ucu ile Qualcomm QNN SDK kullanır. Snapdragon 8 Gen 1 veya daha yenisi gerektirir.</string>
163+
162164
<string name="drawer_web_ui">Web arayüzü</string>
163165

164166
<string name="home_tab_txt_to_img">Txt2Img</string>
@@ -303,6 +305,7 @@
303305
<string name="error_localhost_url">Sunucuya bağlanmak için localhost (127.0.0.1) URL adresi kullanamazsınız.</string>
304306
<string name="error_min_size">Asgari boyut %1$s</string>
305307
<string name="error_max_size">Azami boyut %1$s</string>
308+
<string name="error_qnn_unsupported_resolution">QNN için desteklenmeyen çözünürlük. Geçerli bir çözünürlük seçmek için açılır listeyi kullanın.</string>
306309
<string name="error_download_fail">Yükleme başarısız</string>
307310
<string name="error_generic">İsteğiniz işlenirken bir sorun oluştu. Lütfen daha sonra tekrar deneyin.</string>
308311

@@ -326,6 +329,10 @@
326329
<string name="model_local_custom_title">Yerel özel modeli kullanmak için telefonunuzun depolama alanındaki yerel klasöre yerleştirin.</string>
327330
<string name="model_local_permission_header">İzinler</string>
328331
<string name="model_local_custom_sub_title">Son klasör yapısı şu şekilde olmalıdır::</string>
332+
<string name="model_qnn_custom_sub_title">Gerekli QNN klasör yapısı:</string>
333+
<string name="action_scan_models">Modelleri tara</string>
334+
<string name="model_scan_empty">Belirtilen klasörde model bulunamadı.\n\nHer modelin gerekli dosyalarla kendi alt klasöründe olduğundan emin olun.</string>
335+
<string name="model_scan_found">%1$d model bulundu:</string>
329336
<string name="model_local_path_header">Model yolu</string>
330337
<string name="model_local_path_title">Yerel model klasör yolu</string>
331338
<string name="model_local_path_button">Klasör seç</string>

‎core/localization/src/main/res/values-uk/strings.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,8 @@
159159

160160
<string name="hint_mediapipe_sub_title">Ця конфігурація використовує Google AI MediaPipe та дозволяє запускати генерації Stable Diffusion на вашому телефоні без необхідності підключатися до віддаленого сервера/хмари.</string>
161161

162+
<string name="hint_qnn_sub_title">Ця конфігурація використовує Qualcomm QNN SDK з прискоренням NPU (HTP) та MNN backend для швидкої генерації Stable Diffusion на пристрої. Потрібен Snapdragon 8 Gen 1 або новіший.</string>
163+
162164
<string name="drawer_web_ui">Веб</string>
163165

164166
<string name="home_tab_txt_to_img">Txt2Img</string>
@@ -303,6 +305,7 @@
303305
<string name="error_localhost_url">Ви не можете використовувати localhost (127.0.0.1) URL-адресу для підключення до сервера.</string>
304306
<string name="error_min_size">Мінімальний розмір %1$s</string>
305307
<string name="error_max_size">Максимальний розмір %1$s</string>
308+
<string name="error_qnn_unsupported_resolution">Непідтримувана роздільна здатність для QNN. Використовуйте випадаючий список для вибору коректної роздільної здатності.</string>
306309
<string name="error_download_fail">Помилка під час завантаження</string>
307310
<string name="error_generic">Під час обробки вашого запиту сталася помилка. Повторіть спробу пізніше.</string>
308311

@@ -326,6 +329,10 @@
326329
<string name="model_local_custom_title">Щоб використовувати локальну спеціальну модель, помістіть її в локальну папку в пам’яті телефону.</string>
327330
<string name="model_local_permission_header">Дозволи</string>
328331
<string name="model_local_custom_sub_title">Остаточна структура папок має бути такою:</string>
332+
<string name="model_qnn_custom_sub_title">Потрібна структура папки QNN:</string>
333+
<string name="action_scan_models">Сканувати моделі</string>
334+
<string name="model_scan_empty">Моделі не знайдено у вказаній папці.\n\nПереконайтеся, що кожна модель знаходиться у власній підпапці з необхідними файлами.</string>
335+
<string name="model_scan_found">Знайдено %1$d моделей:</string>
329336
<string name="model_local_path_header">Шлях моделі</string>
330337
<string name="model_local_path_title">Шлях папки локальної моделі</string>
331338
<string name="model_local_path_button">Виберіть папку</string>

‎core/localization/src/main/res/values-zh/strings.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,8 @@
183183

184184
<string name="hint_mediapipe_sub_title">此配置使用 Google AI MediaPipe 运行时，您可以在手机端运行 Stable Diffusion AI 的生成功能，而无需连接至远程服务器/云。</string>
185185

186+
<string name="hint_qnn_sub_title">此配置使用高通 QNN SDK，通过 NPU 加速（HTP）和 MNN 后端在设备上快速生成 Stable Diffusion 图像。需要骁龙 8 Gen 1 或更新版本。</string>
187+
186188
<!-- 抽屉 -->
187189
<string name="drawer_web_ui">网络界面</string>
188190

@@ -347,6 +349,7 @@
347349
<string name="error_localhost_url">您不能使用本地主机 (127.0.0.1)URL 来连接到服务器。</string>
348350
<string name="error_min_size">最小尺寸为 %1$s</string>
349351
<string name="error_max_size">最大尺寸为 %1$s</string>
352+
<string name="error_qnn_unsupported_resolution">QNN 不支持此分辨率。请使用下拉列表选择有效的分辨率。</string>
350353
<string name="error_download_fail">下载失败</string>
351354
<string name="error_generic">处理您的请求时发生错误，请稍后再试。</string>
352355

@@ -375,6 +378,10 @@
375378
<string name="model_local_custom_title">要使用本地自定义模型，请将其放置在手机存储中的本地文件夹。</string>
376379
<string name="model_local_permission_header">权限</string>
377380
<string name="model_local_custom_sub_title">最终的文件夹结构应该是：</string>
381+
<string name="model_qnn_custom_sub_title">QNN所需的文件夹结构：</string>
382+
<string name="action_scan_models">扫描模型</string>
383+
<string name="model_scan_empty">在指定的文件夹中未找到模型。\n\n请确保每个模型都在其自己的子文件夹中，并包含所需的文件。</string>
384+
<string name="model_scan_found">找到 %1$d 个模型：</string>
378385

379386
<!-- 调试菜单 -->
380387
<string name="debug_section_main">调试</string>

‎core/localization/src/main/res/values/strings.xml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@
7676
<string name="srv_type_local_short" translatable="false">ONNX</string>
7777
<string name="srv_type_media_pipe" translatable="false">Local Diffusion Google AI MediaPipe (Beta)</string>
7878
<string name="srv_type_media_pipe_short" translatable="false">MediaPipe</string>
79+
<string name="srv_type_qnn" translatable="false">Local Diffusion Qualcomm QNN (Beta)</string>
80+
<string name="srv_type_qnn_short" translatable="false">QNN</string>
7981
<string name="srv_type_hugging_face" translatable="false">Hugging Face Inference</string>
8082
<string name="srv_type_hugging_face_short" translatable="false">HuggingFace</string>
8183
<string name="srv_type_open_ai" translatable="false">Open AI</string>
@@ -95,6 +97,7 @@
9597
<string name="hint_distilled_cfg_scale">Distilled CFG Scale: %1$s</string>
9698
<string name="hint_model_type">Model type</string>
9799
<string name="hint_vae_text_encoder">VAE / Text Encoder</string>
100+
<string name="hint_runtime">Runtime</string>
98101
<string name="hint_sampler">Sampling method</string>
99102
<string name="hint_scheduler">Scheduler</string>
100103
<string name="hint_adetailer">ADetailer</string>
@@ -183,6 +186,9 @@
183186
<string name="hint_mediapipe_title" translatable="false">Local Diffusion Google AI MediaPipe</string>
184187
<string name="hint_mediapipe_sub_title">This configuration uses Google AI MediaPipe and allows to run Stable Diffusion AI generations on your phone, with no need to connect to remote server/cloud.</string>
185188

189+
<string name="hint_qnn_title" translatable="false">Local Diffusion Qualcomm QNN</string>
190+
<string name="hint_qnn_sub_title">This configuration uses Qualcomm QNN SDK with NPU acceleration (HTP) and MNN backend for fast on-device Stable Diffusion generation. Requires Snapdragon 8 Gen 1 or newer.</string>
191+
186192
<string name="drawer_web_ui">Web UI</string>
187193

188194
<string name="home_tab_txt_to_img">Txt2Img</string>
@@ -215,6 +221,7 @@
215221
<string name="gallery_media_store_banner">You have %1$s photos saved in Download/SDAI</string>
216222
<string name="gallery_info_field_date">Created</string>
217223
<string name="gallery_info_field_type">Type</string>
224+
<string name="gallery_info_field_model">Model</string>
218225
<string name="gallery_info_field_prompt">Prompt</string>
219226
<string name="gallery_info_field_negative_prompt">Negative prompt</string>
220227
<string name="gallery_info_field_size">Size</string>
@@ -295,6 +302,9 @@
295302
<string name="interaction_warning_title">Warning</string>
296303
<string name="interaction_warning_localhost_sub_title">You are trying to connect to localhost (127.0.0.1) server.\n\nIt may not work unless you have ssh tunneling or another port forwarding mechanism set up on your Android device.</string>
297304

305+
<string name="warning_gpu_runtime_title">GPU Runtime Warning</string>
306+
<string name="warning_gpu_runtime_message">GPU runtime warning: The first run at each resolution will be slow as it generates cache files. Subsequent runs will be much faster.</string>
307+
298308
<string name="interaction_delete_generation_title">Delete image</string>
299309
<string name="interaction_delete_generation_sub_title">Are you sure you want to permanently delete this image?</string>
300310

@@ -334,6 +344,7 @@
334344
<string name="error_localhost_url">You can not use localhost (127.0.0.1) URL to connect to server.</string>
335345
<string name="error_min_size">Minimum size is %1$s</string>
336346
<string name="error_max_size">Maximum size is %1$s</string>
347+
<string name="error_qnn_unsupported_resolution">Unsupported resolution for QNN. Use dropdown to select valid resolution.</string>
337348
<string name="error_download_fail">Download failed</string>
338349
<string name="error_generic">Something wrong happened while processing your request, please try again later.</string>
339350

@@ -362,6 +373,10 @@
362373

363374
<string name="model_local_custom_title">To use local custom model, place it to local folder in your phone storage.</string>
364375
<string name="model_local_custom_sub_title">The final folder structure should be:</string>
376+
<string name="model_qnn_custom_sub_title">Required QNN folder structure:</string>
377+
<string name="action_scan_models">Scan for models</string>
378+
<string name="model_scan_empty">No models found in the specified folder.\n\nMake sure each model is in its own subfolder with required files.</string>
379+
<string name="model_scan_found">Found %1$d model(s):</string>
365380

366381
<string name="debug_section_main">Debugging</string>
367382
<string name="debug_section_ld" translatable="false">Local Diffusion</string>

0 commit comments

Comments
 (0)