Enable logographic detection (#2056)

* Trim up some of the data a bit who don't know how to use SEO properly. i.e. possible spamming/poisoning
* Bug fix using incorrect identifier.

Post #2048

NOTE(s):
* I tried for HOURS to get these settings into settings.json *(hence why the require is left in one file)* however RegExp just wouldn't have it no matter what I tried. Existing scripts worked, new scripts did not. If anyone has an idea PRs are welcome to move its location to settings.json.
* DB migration will happen when I get around to it as this is very complicated since script source has to be reread but at least new scripts will have it.

Author: Martii

controllers/scriptStorage.js

@@ -8,6 +8,9 @@ var statusError = require('../libs/debug').statusError;
 var isSecured = require('../libs/debug').isSecured;
 var uaOUJS = require('../libs/debug').uaOUJS;
 
+var rLogographic = require('../libs/debug').rLogographic;
+var logographicDivisor = require('../libs/debug').logographicDivisor;
+
 //
 
 //--- Dependency inclusions
@@ -1198,8 +1201,6 @@ exports.storeScript = function (aUser, aMeta, aBuf, aUpdate, aCallback) {
   var userName = findMeta(aMeta, 'OpenUserJS.author.0.value') || aUser.name;
   var scriptName = null;
   var scriptDescription = null;
-  var thisName = null;
-  var thisDescription = null;
   var hasInvalidKey = null;
 
   async.series([
@@ -1250,8 +1251,7 @@ exports.storeScript = function (aUser, aMeta, aBuf, aUpdate, aCallback) {
       // Check for non-localized presence
       name.forEach(function (aElement, aIndex, aArray) {
         if (!name[aIndex].key) {
-          thisName = aElement.value;
-          scriptName = cleanFilename(thisName, '');
+          scriptName = aElement.value;
         }
       });
 
@@ -1311,8 +1311,7 @@ exports.storeScript = function (aUser, aMeta, aBuf, aUpdate, aCallback) {
       if (description) {
         description.forEach(function (aElement, aIndex, aArray) {
           if (!description[aIndex].key) {
-            thisDescription = aElement.value;
-            scriptDescription = cleanFilename(thisDescription, '');
+            scriptDescription = aElement.value;
           }
         });
       }
@@ -2058,6 +2057,8 @@ exports.storeScript = function (aUser, aMeta, aBuf, aUpdate, aCallback) {
         var s3 = null;
         var now = null;
 
+        var storeDescriptionLength = null;
+
         if (aRemoved) {
           aCallback(new statusError({
             message: 'Script removed permanently.',
@@ -2079,11 +2080,18 @@ exports.storeScript = function (aUser, aMeta, aBuf, aUpdate, aCallback) {
         } else if (!aScript) {
           // New script
           now = new Date();
+
+
+          storeDescriptionLength = settings.scriptSearchQueryStoreMaxDescription;
+          storeDescriptionLength = rLogographic.test(scriptDescription)
+            ? parseInt(storeDescriptionLength / logographicDivisor)
+            : storeDescriptionLength;
+
           aScript = new Script({
-            name: thisName,
+            name: scriptName,
             _description: (
-              thisDescription
-                ? thisDescription.substr(0, settings.scriptSearchQueryStoreMaxDescription).trim()
+              scriptDescription
+                ? scriptDescription.substr(0, storeDescriptionLength).trim()
                 : ''
             ),
             author: aUser.name,
@@ -2123,9 +2131,15 @@ exports.storeScript = function (aUser, aMeta, aBuf, aUpdate, aCallback) {
             }), null);
             return;
           }
+
+          storeDescriptionLength = settings.scriptSearchQueryStoreMaxDescription;
+          storeDescriptionLength = rLogographic.test(scriptDescription)
+            ? parseInt(storeDescriptionLength / logographicDivisor)
+            : storeDescriptionLength;
+
           aScript._description = (
-            thisDescription
-              ? thisDescription.substr(0, settings.scriptSearchQueryStoreMaxDescription).trim()
+            scriptDescription
+              ? scriptDescription.substr(0, storeDescriptionLength).trim()
               : ''
           );
           aScript.meta = aMeta;

libs/debug.js

@@ -77,6 +77,11 @@ uaOUJS = pkg.org + '/' + pkg.version
     + 'OUJS/20131106 ' + pkg.name + '/' + hash;
 exports.uaOUJS = uaOUJS;
 
+//  NOTE: Requires DB migration for changing below settings
+exports.rLogographic = /[\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Hangul}]/u;
+exports.logographicDivisor = 4;
+
+// /NOTE:
 
 // ES6+ in use to eliminate extra property
 class statusError extends Error {

libs/modelParser.js

@@ -6,6 +6,9 @@ var isDev = require('../libs/debug').isDev;
 var isDbg = require('../libs/debug').isDbg;
 var statusError = require('../libs/debug').statusError;
 
+var rLogographic = require('../libs/debug').rLogographic;
+var logographicDivisor = require('../libs/debug').logographicDivisor;
+
 //
 
 //--- Dependency inclusions
@@ -31,6 +34,7 @@ var isFQUrl = require('../libs/helpers').isFQUrl;
 var isSameOrigin = require('../libs/helpers').isSameOrigin;
 var patternHasSameOrigin = require('../libs/helpers').patternHasSameOrigin;
 var scriptStorageLib = require('../libs/scriptStorage').invalidKey;
+var settings = require('../models/settings.json');
 
 
 //--- Configuration inclusions
@@ -316,6 +320,9 @@ var parseScript = function (aScript) {
 
   var matches = null;
 
+  var logographic = null;
+  var storeDescriptionLength = null;
+
   if (!aScript) {
     return;
   }
@@ -339,8 +346,14 @@ var parseScript = function (aScript) {
       }
     });
 
-    if (script.description && script._description && script.description.length > script._description.length) {
-      script.hasLongDescription = true;
+    logographic = rLogographic.test(script.description);
+
+    if (script.description && script._description
+      && script.description.length && script.description.length > logographicDivisor
+        && script.description.length > (logographic
+          ? parseInt(script._description.length / logographicDivisor)
+          : script._description.length)) {
+            script.hasLongDescription = true;
     }
   }