fix: search api use base64 encode

Author: voidmuse-dev

extensions/intellij/src/main/java/com/voidmuse/idea/plugin/protocol/CallJavaHandlerImpl.java

@@ -44,6 +44,7 @@
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileWriter;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.util.concurrent.ConcurrentHashMap;
@@ -53,6 +54,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.nio.charset.StandardCharsets;
 
 import static com.voidmuse.idea.plugin.protocol.CallJavaProtocol.*;
 
@@ -450,25 +452,70 @@ private String getFileContent(Project project, String path) {
 
     // 获取URL内容的实现
     private String getUrlContent(String urlString) {
+        HttpURLConnection connection = null;
+        String result = "";
         try {
             URL url = new URL(urlString);
-            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+            connection = (HttpURLConnection) url.openConnection();
             connection.setRequestMethod("GET");
+            connection.setInstanceFollowRedirects(true);
             connection.setConnectTimeout(10000);
             connection.setReadTimeout(10000);
 
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))) {
+            // 许多站点（含 Cloudflare/Zendesk）会拦截无UA的请求，补充常见请求头
+            connection.setRequestProperty("User-Agent",
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) VoidMuse-IntelliJ/1.0 Chrome/118.0 Safari/537.36");
+            connection.setRequestProperty("Accept",
+                    "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
+            connection.setRequestProperty("Accept-Language", "en-US,en;q=0.9");
+            connection.setRequestProperty("Accept-Encoding", "gzip,deflate");
+            // 使用站点根作为通用 Referer，有助于绕过部分拦截
+            connection.setRequestProperty("Referer", url.getProtocol() + "://" + url.getHost() + "/");
+
+            int status = connection.getResponseCode();
+            InputStream stream = (status >= 200 && status < 300)
+                    ? connection.getInputStream()
+                    : connection.getErrorStream();
+
+            if (stream == null) {
+                throw new java.io.IOException("HTTP " + status + " with empty body");
+            }
+
+            // 处理压缩响应
+            String contentEncoding = connection.getContentEncoding();
+            if (contentEncoding != null) {
+                String enc = contentEncoding.toLowerCase();
+                if (enc.contains("gzip")) {
+                    stream = new java.util.zip.GZIPInputStream(stream);
+                } else if (enc.contains("deflate")) {
+                    stream = new java.util.zip.InflaterInputStream(stream);
+                }
+            }
+
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
                 StringBuilder content = new StringBuilder();
                 String line;
                 while ((line = reader.readLine()) != null) {
                     content.append(line).append("\n");
                 }
-                return content.toString();
+                if (status >= 200 && status < 300) {
+                    LOG.info("Successfully fetched content from URL: " + urlString);
+                    result = content.toString();
+                    
+                } else {
+                    LOG.error("Failed to fetch content from URL: " + urlString + " with HTTP status: " + status);
+                    result =  "Error: HTTP " + status + " " + content.toString();
+                }
             }
         } catch (Exception e) {
             LOG.error("Error getting URL content: " + urlString, e);
-            return "Error: " + e.getMessage();
+            result = "Error: " + e.getMessage();
+        } finally {
+            if (connection != null) {
+                connection.disconnect();
+            }
         }
+        return Base64.encode(result);
     }
 
     // 执行命令的实现

extensions/vscode/src/extension.ts

@@ -445,8 +445,10 @@ export class VoidmuseWebViewProvider implements vscode.WebviewViewProvider {
                         break;
                     case "getUrlContent":
                         try {
-                            response = await getUrlContent(arg.url);
-                            console.log(response);
+                            const urlContent = await getUrlContent(arg.url);
+                            console.log(urlContent);
+                            const encodedContent = Buffer.from(urlContent).toString('base64');
+                            response = encodedContent;
                         } catch (error) {
                             vscode.window.showErrorMessage("get html content error!");
                         }

extensions/vscode/src/services/EmbeddingsSettings.ts

@@ -21,7 +21,7 @@ export class EmbeddingsSettings {
 
     /**
      * function: 获取embeddings的模型名称，配置内容：
-     *         "global:embeddingModels": "[{\"key\":\"1752222055608\",\"name\":\"gte-Qwen2-1.5B-instruct-f16:latest\",\"provider\":\"OpenAI\",\"enabled\":true,\"modelId\":\"gte-Qwen2-1.5B-instruct-f16:latest\",\"apiKey\":\"~\",\"baseUrl\":\"https://lpfm2-ollama.yy.com\",\"isCustomModel\":true}]",
+     *         "global:embeddingModels": "[{\"key\":\"1752222055608\",\"name\":\"gte-Qwen2-1.5B-instruct-f16:latest\",\"provider\":\"OpenAI\",\"enabled\":true,\"modelId\":\"gte-Qwen2-1.5B-instruct-f16:latest\",\"apiKey\":\"~\",\"baseUrl\":\"xxx\",\"isCustomModel\":true}]",
         "global:selectedEmbeddingModel": "1752222055608",
         "global:isAutoEmbedding": "false",
      * 遍历embeddingModels，转换为json对象，取enabled=true的name

gui/src/api/IdeaAsyncMgr.ts

@@ -52,7 +52,7 @@ interface CallbackMessage {
 const callJavaCallback = async (message: string): Promise<void> => {
     try {
         // message is json string data, convert to corresponding object
-        console.log(`callJavaCallback message: ${message}`);
+        console.log(`callJavaCallback message: ${message.substring(0, 100)}`);
         const parsedMessage: CallbackMessage = JSON.parse(message); // Convert json string to object
         if (parsedMessage.requestId && IdeaAsyncMgr.pendingRequests.has(parsedMessage.requestId)) {
             const { onSuccess, onFailure } = IdeaAsyncMgr.pendingRequests.get(parsedMessage.requestId)!;

gui/src/api/implementations/IntelliJDE.ts

@@ -564,7 +564,8 @@ export class IntelliJIDE implements IDEInterface {
         IdeaAsyncMgr.sendMessage(param,
           function onSuccess(response) {
             console.log('IntelliJ getUrlContent callback success:', url);
-            resolve(response);
+            const decodedResponse = base64Decode(response);
+            resolve(decodedResponse);
           },
           function onFailure(error_code, error_message) {
             console.error('IntelliJ getUrlContent callback failed:', error_code, error_message);

gui/src/api/implementations/VscodeIDE.ts

@@ -551,7 +551,8 @@ export class VscodeIDE implements IDEInterface {
         vscodeMgr.sendMessage(param,
           function onSuccess(response) {
             console.log('vscode getUrlContent callback success:', url);
-            resolve(response);
+            const decodedResponse = base64Decode(response);
+            resolve(decodedResponse);
           },
           function onFailure(error_code, error_message) {
             console.error('vscode getUrlContent callback failed:', error_code, error_message);

gui/src/services/search/WebContentExtractor.ts

@@ -103,8 +103,8 @@ export class WebContentExtractor {
 
         try {
             // Use allorigins proxy service directly
-            const html = await this.getContentFromProxy(params.url);
-            // const html = await this.getContentFromIDE(params.url);
+            // const html = await this.getContentFromProxy(params.url);
+            const html = await this.getContentFromIDE(params.url);
 
             // Parse HTML content
             const result = this.parseHtmlContent(html, params.url, maxLength);