tab navigation improvements, search updates, and additional stemming language support

Author: pragmar

README.md

@@ -21,7 +21,6 @@ The InterroBot plugin ecosystem is designed for power users. Whether you're buil
 InterroBot hosts an iframe of your webpage and exposes an API from which you can pull data down for analysis.
 
 If you're familiar with vanilla TypeScript or JavaScript, creating a custom plugin script for InterroBot is remarkably straight forward. First you start with a [bare-bones HTML file](https://raw.githubusercontent.com/interrobot/interrobot-plugin/refs/heads/master/examples/vanillajs/basic.html) and a script extending the Plugin base class.
-
 ```javascript
 // TypeScript vs. JavaScript, both are fine. See examples.
 import { Plugin } from "./src/ts/core/plugin";
@@ -48,13 +47,15 @@ Plugin.initialize(BasicExamplePlugin);
 
 BasicExamplePlugin will not do much at this point, but it will load and run the default `index()` behavior.
 You can, of course, override the default `index()` behavior, rendering your page however you wish.
-
 ```javascript
 protected async index() {
+
     // add your form and supporting HTML
     this.render(`<div>HTML</div>`);
+
     // initialize the plugin within InterroBot, from within iframe
     await this.initData({}, []);
+
     // add handlers to the form
     const button = document.querySelector("button");
     button.addEventListener("click", async (ev) => {
@@ -65,10 +66,9 @@ protected async index() {
 
 The `process()` method called above would be where you process data. Here a query is executed on
 the crawl index, and each result run through the exampleResultsHandler.
-
-
 ```javascript
 protected async process() {
+
     // gather title words and running counts with a result handler
     const titleWords: Map<string, number> = new Map<string, number>();
     let resultsMap: Map<number, SearchResult>;
@@ -81,24 +81,36 @@ protected async process() {
 
     // projectId comes for free as a member of Plugin
     const projectId = this.getProjectId();
+
     // build a query, these are exactly as you'd type them into InterroBot search
     const freeQueryString = "headers: text/html";
-    // pipe delimited fields you want retrieved
+
+
     // id and url come with the base model, everything else costs time
-    const fields = "name";
+    // here, I just grab the "name" field
     let internalHtmlPagesQuery = new InterroBot.Core.SearchQuery({
         project: projectId,
         query: freeQueryString,
-        fields: fields,
+        fields: ["name"],
         type: InterroBot.Core.SearchQueryType.Any,
         includeExternal: false,
         includeNoRobots: false,
     });
 
     // run each SearchResult through its handler, and we're done processing
-    await InterroBot.Core.Search.execute(internalHtmlPagesQuery, this.resultsMap, async (result) => {
-        await exampleResultHandler(result, titleWords);
-    }, true, false, "Processing…");
+    await InterroBot.Core.Search.execute(
+        internalHtmlPagesQuery,
+        this.resultsMap,
+        async (result) => {
+            await exampleResultHandler(result, titleWords);
+        },
+        {
+            paginate: true,
+            showProgress: false,
+            progressMessage: "Processing…"
+        }
+    );
+
     // call for HTML presentation of titleWords with processing complete
     await this.report(titleWords);
 }
@@ -164,4 +176,4 @@ Retrieves a list of crawls using the Plugin API.
 MPL 2.0, with exceptions. This repo contains JavaScript to TypeScript ports and a Markdown library based on existing code, all contained within `./src/lib`. As they arrived under existing licenses, they will remain under those.
 
 * *Typo.js*: TypeScript port continues under the original [Modified BSD License](https://raw.githubusercontent.com/cfinke/Typo.js/master/license.txt).
-* *Snowball.js*: TypeScript port continues under the original [MPL 1.1](https://raw.githubusercontent.com/fortnightlabs/snowball-js/master/LICENSE) license.
+* *Snowball.js*: TypeScript port continues under the original [MPL 1.1](https://raw.githubusercontent.com/fortnightlabs/snowball-js/master/LICENSE) license.

dist/js/commonjs/core/api.d.ts

@@ -2,14 +2,14 @@
  * Enumeration for different types of search queries.
  */
 declare enum SearchQueryType {
-    Page = 0,
-    Asset = 1,
-    Any = 2
+    Page = "page",
+    Asset = "asset",
+    Any = "any"
 }
 interface SearchQueryParams {
     project: number;
     query: string;
-    fields: string;
+    fields: string[];
     type: SearchQueryType;
     includeExternal?: boolean;
     includeNoRobots?: boolean;
@@ -34,6 +34,11 @@ interface SearchResultJson {
     assets?: string[];
     origin?: string;
 }
+interface SearchExecuteOptions {
+    paginate?: boolean;
+    showProgress?: boolean;
+    progressMessage?: string;
+}
 interface CrawlParams {
     id: number;
     project: number;
@@ -71,7 +76,7 @@ declare class PluginData {
     private project;
     /**
      * Creates an instance of PluginData.
-     * @param params - The plugin data parameters.
+     * @param params - Configuration object containing projectId, meta, defaultData, and autoformInputs
      */
     constructor(params: PluginDataParams);
     /**
@@ -116,15 +121,15 @@ declare class SearchQuery {
     private static readonly validSorts;
     readonly project: number;
     readonly query: string;
-    readonly fields: string;
+    readonly fields: string[];
     readonly type: SearchQueryType;
     readonly includeExternal: boolean;
     readonly includeNoRobots: boolean;
     readonly sort: string;
     readonly perPage: number;
     /**
      * Creates an instance of SearchQuery.
-     * @param params - The search query parameters.
+     * @param params - Configuration object containing project, query, fields, type, includeExternal, and includeNoRobots
      */
     constructor(params: SearchQueryParams);
     /**
@@ -138,13 +143,13 @@ declare class Search {
     private static resultsHaystackCacheKey;
     /**
      * Executes a search query.
-     * @param query - The search query to execute.
-     * @param existingResults - Map of existing results.
-     * @param processingMessage - Message to display during processing.
-     * @param resultHandler - Function to handle each search result.
-     * @returns A promise that resolves to a boolean indicating if results were from cache.
+     * @param query - The search query to execute
+     * @param resultsMap - Map of existing results
+     * @param resultHandler - Function to handle each search result
+     * @param options - Optional configuration for pagination, progress display, and custom messages
+     * @returns A promise that resolves to a boolean indicating if results were from cache
      */
-    static execute(query: SearchQuery, existingResults: Map<number, SearchResult>, resultHandler: any, deep?: boolean, quiet?: boolean, processingMessage?: string): Promise<boolean>;
+    static execute(query: SearchQuery, resultsMap: Map<number, SearchResult>, resultHandler: (result: SearchResult) => Promise<void>, options?: SearchExecuteOptions): Promise<boolean>;
     /**
      * Sleeps for the specified number of milliseconds.
      * @param millis - The number of milliseconds to sleep.
@@ -241,7 +246,7 @@ declare class Crawl {
     report?: any;
     /**
      * Creates an instance of Crawl.
-     * @param params - The crawl parameters.
+     * @param params - Configuration object containing id, project, created, modified, complete, time, and report
      */
     constructor(params: CrawlParams);
     /**
@@ -276,7 +281,7 @@ declare class Project {
     static readonly urlDeprectionWarning: string;
     /**
      * Creates an instance of Project.
-     * @param params - The project parameters.
+     * @param params - Configuration object containing id, created, modified, name, type, url, urls, and imageDataUri
      */
     constructor(params: ProjectParams);
     /**
@@ -303,4 +308,4 @@ declare class Project {
      */
     static getApiCrawls(project: number): Promise<Crawl[]>;
 }
-export { Project, Crawl, SearchQueryType, SearchQuery, Search, SearchResult, SearchResultJson, PluginData };
+export { Project, ProjectParams, Crawl, CrawlParams, SearchQueryType, SearchQuery, SearchQueryParams, Search, SearchExecuteOptions, SearchResult, SearchResultJson, PluginData, PluginDataParams };

dist/js/commonjs/core/api.js

@@ -10,30 +10,18 @@ const plugin_js_1 = require("./plugin.js");
  */
 var SearchQueryType;
 (function (SearchQueryType) {
-    SearchQueryType[SearchQueryType["Page"] = 0] = "Page";
-    SearchQueryType[SearchQueryType["Asset"] = 1] = "Asset";
-    SearchQueryType[SearchQueryType["Any"] = 2] = "Any";
+    SearchQueryType["Page"] = "page";
+    SearchQueryType["Asset"] = "asset";
+    SearchQueryType["Any"] = "any";
 })(SearchQueryType || (SearchQueryType = {}));
 exports.SearchQueryType = SearchQueryType;
-var SearchQuerySortField;
-(function (SearchQuerySortField) {
-    SearchQuerySortField[SearchQuerySortField["Id"] = 0] = "Id";
-    SearchQuerySortField[SearchQuerySortField["Time"] = 1] = "Time";
-    SearchQuerySortField[SearchQuerySortField["Status"] = 2] = "Status";
-    SearchQuerySortField[SearchQuerySortField["Url"] = 3] = "Url";
-})(SearchQuerySortField || (SearchQuerySortField = {}));
-var SearchQuerySortDirection;
-(function (SearchQuerySortDirection) {
-    SearchQuerySortDirection[SearchQuerySortDirection["Ascending"] = 0] = "Ascending";
-    SearchQuerySortDirection[SearchQuerySortDirection["Descending"] = 1] = "Descending";
-})(SearchQuerySortDirection || (SearchQuerySortDirection = {}));
 /**
  * Container for plugin settings
  */
 class PluginData {
     /**
      * Creates an instance of PluginData.
-     * @param params - The plugin data parameters.
+     * @param params - Configuration object containing projectId, meta, defaultData, and autoformInputs
      */
     constructor(params) {
         var _a;
@@ -378,15 +366,21 @@ exports.PluginData = PluginData;
 class SearchQuery {
     /**
      * Creates an instance of SearchQuery.
-     * @param params - The search query parameters.
+     * @param params - Configuration object containing project, query, fields, type, includeExternal, and includeNoRobots
      */
     constructor(params) {
         var _a, _b, _c;
         this.includeExternal = true;
         this.includeNoRobots = false;
         this.project = params.project;
         this.query = params.query;
-        this.fields = params.fields;
+        // backcompat <=0.17 (piped string handling)
+        if (typeof params.fields === "string") {
+            this.fields = params.fields.split("|");
+        }
+        else {
+            this.fields = params.fields;
+        }
         this.type = params.type;
         this.includeExternal = (_a = params.includeExternal) !== null && _a !== void 0 ? _a : true;
         this.includeNoRobots = (_b = params.includeNoRobots) !== null && _b !== void 0 ? _b : false;
@@ -403,7 +397,7 @@ class SearchQuery {
      * @returns A string representing the cache key.
      */
     getHaystackCacheKey() {
-        return `${this.project}~${this.fields}~${this.type}~${this.includeExternal}~${this.includeNoRobots}`;
+        return `${this.project}~${this.fields.join("|")}~${this.type}~${this.includeExternal}~${this.includeNoRobots}`;
     }
 }
 exports.SearchQuery = SearchQuery;
@@ -412,35 +406,36 @@ SearchQuery.validSorts = ["?", "id", "-id", "time", "-time", "status", "-status"
 class Search {
     /**
      * Executes a search query.
-     * @param query - The search query to execute.
-     * @param existingResults - Map of existing results.
-     * @param processingMessage - Message to display during processing.
-     * @param resultHandler - Function to handle each search result.
-     * @returns A promise that resolves to a boolean indicating if results were from cache.
+     * @param query - The search query to execute
+     * @param resultsMap - Map of existing results
+     * @param resultHandler - Function to handle each search result
+     * @param options - Optional configuration for pagination, progress display, and custom messages
+     * @returns A promise that resolves to a boolean indicating if results were from cache
      */
-    static async execute(query, existingResults, resultHandler, deep = false, quiet = true, processingMessage = "Processing...") {
+    static async execute(query, resultsMap, resultHandler, options) {
         const timeStart = new Date().getTime();
+        const { paginate = false, showProgress = true, progressMessage = "Processing..." } = options !== null && options !== void 0 ? options : {};
         // Promise<boolean> returned is a from-cache flag, true if cached
-        if (query.getHaystackCacheKey() === Search.resultsHaystackCacheKey && existingResults) {
-            const resultTotal = existingResults.size;
+        if (query.getHaystackCacheKey() === Search.resultsHaystackCacheKey && resultsMap) {
+            const resultTotal = resultsMap.size;
             // reuse api reuslts
             // print something to screen to inform user of operation
             // this is a blitz, doesn't get the http request breathing room of api http requests
             // anyways, paint first, then saturate cpu
-            if (quiet === false) {
-                const eventStart = new CustomEvent("ProcessingMessage", { detail: { action: "set", message: processingMessage } });
+            if (showProgress === true) {
+                const eventStart = new CustomEvent("ProcessingMessage", { detail: { action: "set", message: progressMessage } });
                 document.dispatchEvent(eventStart);
             }
             // give main thread a short break to render progress
             await Search.sleep(16);
             // note for of loop with sleep mod 100 works, looks smooth, but slows the operation by > 20%
             // this is faster, but it can't paint progress well as it can saturate the main thread
             let i = 0;
-            await existingResults.forEach(async (result, resultId) => {
+            await resultsMap.forEach(async (result, resultId) => {
                 await resultHandler(result);
             });
             plugin_js_1.Plugin.logTiming(`Processed ${resultTotal.toLocaleString()} search result(s)`, new Date().getTime() - timeStart);
-            if (quiet === false) {
+            if (showProgress === true) {
                 const msg = { detail: { action: "clear" } };
                 const eventFinished = new CustomEvent("ProcessingMessage", msg);
                 document.dispatchEvent(eventFinished);
@@ -455,9 +450,9 @@ class Search {
             "project": query.project,
             "query": query.query,
             "external": query.includeExternal,
-            "type": SearchQueryType[query.type].toLowerCase(),
+            "type": query.type,
             "offset": 0,
-            "fields": query.fields.split("|"),
+            "fields": query.fields,
             "norobots": query.includeNoRobots,
             "sort": query.sort,
             "perpage": query.perPage,
@@ -470,9 +465,13 @@ class Search {
             const result = results[i];
             await Search.handleResult(result, resultTotal, resultHandler);
         }
-        while (responseJson["__meta__"]["results"]["pagination"]["nextOffset"] !== null && deep === true) {
+        while (responseJson["__meta__"]["results"]["pagination"]["nextOffset"] !== null && paginate === true) {
             const next = responseJson["__meta__"]["results"]["pagination"]["nextOffset"];
             kwargs["offset"] = next;
+            if (query.sort === "?" && next > 0) {
+                console.warn("Random sort (?) with pagination generates fresh randomness on each page. " +
+                    "Consider maxing perpage (100) and using 1 page of results when sampling.");
+            }
             responseJson = await plugin_js_1.Plugin.postApiRequest("GetResources", kwargs);
             results = responseJson.results;
             for (let i = 0; i < results.length; i++) {
@@ -633,7 +632,7 @@ SearchResult.wordWhitespaceRe = /\s+/g;
 class Crawl {
     /**
      * Creates an instance of Crawl.
-     * @param params - The crawl parameters.
+     * @param params - Configuration object containing id, project, created, modified, complete, time, and report
      */
     constructor(params) {
         this.id = -1;
@@ -690,7 +689,7 @@ exports.Crawl = Crawl;
 class Project {
     /**
      * Creates an instance of Project.
-     * @param params - The project parameters.
+     * @param params - Configuration object containing id, created, modified, name, type, url, urls, and imageDataUri
      */
     constructor(params) {
         this.id = -1;