Merge pull request #10 from dgw-dev/dw_main_dev

dgw-dev · web-flow · commit fd810b6c57cd · 2025-05-17T17:35:31.000+01:00
request size control
diff --git a/DbLocalizer.Tests/appsettings.Test.json b/DbLocalizer.Tests/appsettings.Test.json
@@ -8,14 +8,14 @@
     }
   },
   "ConnectionStrings": {
-    "LocalizationTest": "Server=localhost;Initial Catalog=LocalizationTest;User Id={0};Password={1};MultipleActiveResultSets=True;TrustServerCertificate=True"
+    "LocalizationTest": "Server={0};Initial Catalog={1};User Id={2};Password={3};MultipleActiveResultSets=True;TrustServerCertificate=True"
   },
   "UseScheduler": "false",
   "ScheduledExportCron": "0 0 6 ? * MON *",
   "ScheduledImportPurgeCron": "0 0 6 ? * FRI *",
   "ImportMaxTablesPerFile": "20",
   "ExportMaxTablesPerFile": "20",
-  "MaxRowsPerFile": "1000",
+  "MaxRowsPerTable": "1000",
   "ExportLookbackInDays": "5000",
   "MaximumRequestSize": "10485760", // 10MB in bytes
   "DataToPurgeInDays": "60",
diff --git a/DbLocalizer/appsettings.Development.json b/DbLocalizer/appsettings.Development.json
@@ -8,14 +8,14 @@
     }
   },
   "ConnectionStrings": {
-    "LocalizationTest": "Server=localhost;Initial Catalog=LocalizationTest;User Id={0};Password={1};MultipleActiveResultSets=True;TrustServerCertificate=True"
+    "LocalizationTest": "Server={0};Initial Catalog={1};User Id={2};Password={3};MultipleActiveResultSets=True;TrustServerCertificate=True"
   },
   "UseScheduler": "false",
   "ScheduledExportCron": "0 0 6 ? * MON *",
   "ScheduledImportPurgeCron": "0 0 6 ? * FRI *",
   "ImportMaxTablesPerFile": "20",
   "ExportMaxTablesPerFile": "20",
-  "MaxRowsPerFile": "1000",
+  "MaxRowsPerTable": "500",
   "ExportLookbackInDays": "5000",
   "MaximumRequestSize": "10485760", // 10MB in bytes
   "DataToPurgeInDays": "60",
diff --git a/DbLocalizer/appsettings.json b/DbLocalizer/appsettings.json
@@ -8,14 +8,14 @@
     }
   },
   "ConnectionStrings": {
-    "LocalizationTest": "Server=localhost;Initial Catalog=LocalizationTest;User Id={0};Password={1};MultipleActiveResultSets=True;TrustServerCertificate=True"
+    "LocalizationTest": "Server={0};Initial Catalog={1};User Id={2};Password={3};MultipleActiveResultSets=True;TrustServerCertificate=True"
   },
   "UseScheduler": "false",
   "ScheduledExportCron": "0 0 6 ? * MON *",
   "ScheduledImportPurgeCron": "0 0 6 ? * FRI *",
   "ImportMaxTablesPerFile": "20",
   "ExportMaxTablesPerFile": "20",
-  "MaxRowsPerFile": "1000",
+  "MaxRowsPerTable": "1000",
   "ExportLookbackInDays": "5000",
   "MaximumRequestSize": "10485760", // 10MB in bytes
   "DataToPurgeInDays": "60",
diff --git a/DbLocalizer/logs.db b/DbLocalizer/logs.db
diff --git a/Entities/AppSettings.cs b/Entities/AppSettings.cs
@@ -7,7 +7,7 @@ public class AppSettings
         public string ScheduledImportPurgeCron { get; set; }
         public string ImportMaxTablesPerFile { get; set; }
         public string ExportMaxTablesPerFile { get; set; }
-        public string MaxRowsPerFile { get; set; }
+        public string MaxRowsPerTable { get; set; }
         public string ExportLookbackInDays { get; set; }
         public string MaximumRequestSize { get; set; }
         public string DataToPurgeInDays { get; set; }
diff --git a/Entities/DAL/DbCredentialsManager.cs b/Entities/DAL/DbCredentialsManager.cs
@@ -10,7 +10,7 @@ public class DbCredentialsManager
         private readonly Dictionary<string, DbCredentials> _dbCredentials = new Dictionary<string, DbCredentials>()
         {
             {
-                "LocalizationTest", new DbCredentials() { Username = "LocalizationUser", Password = "@LocalizationTest#"}
+                "LocalizationTest", new DbCredentials() { ServerName = "localhost", DatabaseName = "LocalizationTest", Username = "LocalizationUser", Password = "@LocalizationTest#"}
             }
         };
         public DbCredentials GetDbCredentials(string connectionStringName)
@@ -21,6 +21,8 @@ public DbCredentials GetDbCredentials(string connectionStringName)
 
     public class DbCredentials
     {
+        public string DatabaseName { get; set; }
+        public string ServerName { get; set; }
         public string Username { get; set; }
         public string Password { get; set; }
     }
diff --git a/Entities/Databases.cs b/Entities/Databases.cs
@@ -41,9 +41,14 @@ private void BuildDatabases()
                 string connectionString = _config.GetConnectionString(db.ConnectionStringName);
                 DbCredentialsManager dbCredentialsManager = new DbCredentialsManager();
                 DbCredentials dbCredentials = dbCredentialsManager.GetDbCredentials(db.ConnectionStringName);
-                if (dbCredentials != null && !string.IsNullOrEmpty(dbCredentials.Username) && !string.IsNullOrEmpty(dbCredentials.Password) && !string.IsNullOrEmpty(connectionString))
+                if (dbCredentials != null
+                    && !string.IsNullOrEmpty(dbCredentials.ServerName)
+                    && !string.IsNullOrEmpty(dbCredentials.DatabaseName)
+                    && !string.IsNullOrEmpty(dbCredentials.Username) 
+                    && !string.IsNullOrEmpty(dbCredentials.Password) 
+                    && !string.IsNullOrEmpty(connectionString))
                 {
-                    connectionString = string.Format(connectionString, dbCredentials.Username, dbCredentials.Password);
+                    connectionString = string.Format(connectionString, dbCredentials.ServerName, dbCredentials.DatabaseName, dbCredentials.Username, dbCredentials.Password);
                     db.ConnectionStringValue = connectionString;
                     db.CanConnect = DatabaseCanConnect(connectionString);
                     _sqlSchemaBuilder.BuildDatabaseSchemaFiles(db);
diff --git a/Entities/Plugins/TranslationManagement/Smartling/ISmartlingExportFileProcessor.cs b/Entities/Plugins/TranslationManagement/Smartling/ISmartlingExportFileProcessor.cs
@@ -14,7 +14,7 @@ public interface ISmartlingExportFileProcessor
         string ExportType { get; set; }
         ICacheManager CacheManager { get; set; }
         Dictionary<string, List<DataSet>> DatabaseUpdatesForApp { get; set; }
-        T ProcessExportData<T>(string exportLookbackInDays, int exportMaxTablesPerFile, int maxRowsPerFile, CancellationToken ct, Database db);
+        T ProcessExportData<T>(string exportLookbackInDays, int exportMaxTablesPerFile, int maxRowsPerTable, CancellationToken ct, Database db);
         MultipartFormDataContent GetCultureContent(SmartlingExportFile culturePackage, List<SmartlingLocaleId> localeIds, string callbackUrl = default(string));
     }
 }
diff --git a/Entities/Plugins/TranslationManagement/Smartling/SmartlingExportFileProcessor.cs b/Entities/Plugins/TranslationManagement/Smartling/SmartlingExportFileProcessor.cs
@@ -36,7 +36,7 @@ public SmartlingExportFileProcessor(
             _smartlingConfiguration = smartlingConfig;
         }
 
-        public T ProcessExportData<T>(string exportLookbackInDays, int exportMaxTablesPerFile, int maxRowsPerFile, CancellationToken ct, Database db)
+        public T ProcessExportData<T>(string exportLookbackInDays, int exportMaxTablesPerFile, int maxRowsPerTable, CancellationToken ct, Database db)
         {
             Dictionary<string, ExportTable> exportTables = new Dictionary<string, ExportTable>();
             Dictionary<string, ExportTable> allTablesToExport = new Dictionary<string, ExportTable>();
diff --git a/Entities/Plugins/TranslationManagement/Smartling/SmartlingExportUtility.cs b/Entities/Plugins/TranslationManagement/Smartling/SmartlingExportUtility.cs
@@ -8,6 +8,7 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Entities;
 using static Entities.Plugins.TranslationManagement.Smartling.SmartlingFileData;
 
 namespace Entities.Plugins.TranslationManagement.Smartling
@@ -54,7 +55,7 @@ protected override async Task ProcessExport(CancellationToken ct, List<string> c
             {
                 string packageId = Guid.NewGuid().ToString();
 
-                dtmData = _fileProcessor.ProcessExportData<SmartlingAppData>(ExportLookbackInDays, ExportMaxTablesPerFile, MaxRowsPerFile, ct, db);
+                dtmData = _fileProcessor.ProcessExportData<SmartlingAppData>(ExportLookbackInDays, ExportMaxTablesPerFile, MaxRowsPerTable, ct, db);
 
                 if (dtmData.FileData != null && dtmData.FileData.Tables.Count > 0)
                 {
@@ -112,60 +113,44 @@ public async Task BuildPackageAsync(SmartlingAppData appData, string packageId,
             }
         }
 
-        public Dictionary<string, SmartlingExportFile> BuildDtmFileDataCollection(SmartlingFileData dtmCultureData, int exportMaxTablesPerFile, int maxRowsPerFile)
+        public static Dictionary<string, SmartlingExportFile> BuildDtmFileDataCollection(SmartlingFileData dtmCultureData, int exportMaxTablesPerFile, int maxRowsPerTable)
         {
-            //A dictionary to hold all sub files per file
-            Dictionary<string, SmartlingExportFile> subFiles = new Dictionary<string, SmartlingExportFile>();
-
-            //A dictionary to hold all sub data per file
-            Dictionary<string, SmartlingFileData> subData = new Dictionary<string, SmartlingFileData>();
+            if (dtmCultureData != null && dtmCultureData.Tables != null && dtmCultureData.Tables.Any())
+            {
+                //A dictionary to hold all sub files per file
+                Dictionary<string, SmartlingExportFile> subFiles = new Dictionary<string, SmartlingExportFile>();
 
-            //first we need to divide the data based on the exportMaxTablesPerFile value and then the maxRowsPerFile value
-            IEnumerable<IEnumerable<Table>> chunkTables = dtmCultureData?.Tables?.AsEnumerable().ToChunks(exportMaxTablesPerFile)
-                  .Select(rows => rows.ToList());
+                //A dictionary to hold all row chunks per table file
+                Dictionary<string, List<HashSet<Row>>> tabelRowChunks = new Dictionary<string, List<HashSet<Row>>>();
 
-            int tableListCount = 0;
-            if ((bool)chunkTables?.Any())
-            {
-                foreach (IEnumerable<Table> tableList in chunkTables)
+                int tableCount = 0;
+                foreach (Table table in dtmCultureData.Tables)
                 {
-                    SmartlingFileData subFile = new SmartlingFileData();
-                    subFile.smartling = dtmCultureData.smartling;
-                    subFile.GlobalizationMetaData = dtmCultureData.GlobalizationMetaData;
-                    subFile.Tables = new List<Table>();
-                    foreach (Table table in tableList)
-                    {
-                        IEnumerable<IEnumerable<Row>> chunkRows = table.Rows?.AsEnumerable().ToChunks(maxRowsPerFile)
-                              .Select(rows => rows.ToHashSet());
+                    //Get rows based on the max number of rows we're allowed per table
+                    List<HashSet<Row>> chunkRows = SplitRowList<Row>(table.Rows, maxRowsPerTable);
 
-                        if (chunkRows != null)
+                    if (chunkRows != null)
+                    {
+                        foreach (IEnumerable<Row> chunk in chunkRows)
                         {
-                            //create a new file for each chunk of rows
-                            foreach (IEnumerable<Row> chunk in chunkRows)
-                            {
+                            List<HashSet<Row>> rows = new List<HashSet<Row>>() { chunk as HashSet<Row> };
 
-                                Table subTable = new Table()
-                                {
-                                    PrimaryKeyName = table.PrimaryKeyName,
-                                    BaseTable = table.BaseTable,
-                                    FullBaseTableName = table.FullBaseTableName,
-                                    LocalizedTable = table.LocalizedTable,
-                                    FullLocalizedTableName = table.FullLocalizedTableName,
-                                    TableSchema = table.TableSchema,
-                                    SystemColumns = table.SystemColumns,
-                                    SourceLanguageColumns = table.SourceLanguageColumns,
-                                    TranslatedLanguageColumns = table.TranslatedLanguageColumns,
-
-                                    Rows = chunk as HashSet<Row>,
-                                };
-                                subFile.Tables.Add(subTable);
+                            if (!tabelRowChunks.TryGetValue(table.FullBaseTableName, out List<HashSet<Row>> _))
+                            {
+                                tabelRowChunks.Add(table.FullBaseTableName, rows);
+                            }
+                            else
+                            {
+                                tabelRowChunks[table.FullBaseTableName].AddRange(rows);
                             }
                         }
                     }
-                    subData.Add(subFile.GlobalizationMetaData.PackageId + "_" + dtmCultureData.GlobalizationMetaData.Locale + "_" + tableListCount.ToString(), subFile);
-                    tableListCount++;
+                    tableCount++;
                 }
 
+                //split data into sub data objects
+                Dictionary<string, SmartlingFileData> subData = GetSubData(dtmCultureData, tabelRowChunks);
+
                 //now we need to create ExportFile objects for each sub data object
                 if (subData?.Count > 0)
                 {
@@ -175,27 +160,30 @@ public Dictionary<string, SmartlingExportFile> BuildDtmFileDataCollection(Smartl
                         subFiles.Add(fileCollection.Key, AddFile(fileCollection.Key, fileCollection.Value, dtmCultureData.GlobalizationMetaData));
                     }
                 }
+
+                return subFiles;
             }
 
-            return subFiles;
+            return new Dictionary<string, SmartlingExportFile>();
         }
 
-        public virtual void AddDtmPackage(ref Dictionary<string, SmartlingExportFile> dtmPackage, SmartlingFileData fileData, string packageId)
+        public virtual void AddDtmPackage(ref Dictionary<string, SmartlingExportFile> exportPackage, SmartlingFileData fileData, string packageId)
         {
             try
             {
-                Dictionary<string, SmartlingExportFile> subFiles = BuildDtmFileDataCollection(fileData, ExportMaxTablesPerFile, MaxRowsPerFile);
+                Dictionary<string, SmartlingExportFile> subFiles = BuildDtmFileDataCollection(fileData, ExportMaxTablesPerFile, MaxRowsPerTable);
 
                 if (subFiles?.Count > 0)
                 {
                     foreach (KeyValuePair<string, SmartlingExportFile> subFile in subFiles)
                     {
-                        dtmPackage.Add(subFile.Key, subFile.Value);
+                        exportPackage.Add(subFile.Key, subFile.Value);
                     }
                 }
                 else
                 {
-                    dtmPackage.Add(packageId, new SmartlingExportFile(fileData, fileData.GlobalizationMetaData));
+                    fileData.GlobalizationMetaData.LastFile = true;
+                    exportPackage.Add(packageId, new SmartlingExportFile(fileData, fileData.GlobalizationMetaData));
                 }
             }
             catch (Exception ex)
@@ -212,5 +200,68 @@ public static SmartlingExportFile AddFile(string fileName, SmartlingFileData job
             exportFile.FileName = fileName + ".json";
             return exportFile;
         }
+
+        private static Dictionary<string, SmartlingFileData> GetSubData(SmartlingFileData dtmCultureData, Dictionary<string, List<HashSet<Row>>> tabelRowChunks)
+        {
+            // Find the entry with the largest number of HashSet<Row> in its value list
+            var maxEntry = tabelRowChunks
+                .OrderByDescending(kvp => kvp.Value?.Count ?? 0)
+                .FirstOrDefault();
+
+            //we need to know how many files we will need to create for each table
+            int totalFilesNeeded = maxEntry.Value?.Count ?? 0;
+
+            Dictionary<string, SmartlingFileData> subData = new Dictionary<string, SmartlingFileData>();
+
+            int fileCount = 0;
+            
+            while (fileCount < totalFilesNeeded)
+            {
+                SmartlingFileData subDataFile = new SmartlingFileData();
+                subDataFile.Tables = new List<Table>();
+                subDataFile.smartling = dtmCultureData.smartling;
+                subDataFile.GlobalizationMetaData = dtmCultureData.GlobalizationMetaData;
+                string fileName = $"{dtmCultureData.GlobalizationMetaData.PackageId}_{fileCount}";
+
+                foreach (Table table in dtmCultureData.Tables)
+                {
+                    var rowSet = tabelRowChunks.FirstOrDefault(l => l.Key == table.FullBaseTableName).Value[fileCount];
+                    if (rowSet != null && rowSet.Count > fileCount)
+                    {
+                        // Create a new table object for each chunk
+                        Table newTable = new Table()
+                        {
+                            Rows = rowSet,
+                            FullBaseTableName = table.FullBaseTableName,
+                            LocalizedTable = table.LocalizedTable,
+                            PrimaryKeyName = table.PrimaryKeyName,
+                            SystemColumns = table.SystemColumns,
+                            SourceLanguageColumns = table.SourceLanguageColumns,
+                            TranslatedLanguageColumns = table.TranslatedLanguageColumns
+                        };
+                        subDataFile.Tables.Add(newTable);
+                    }
+                }
+                subData.Add(fileName, subDataFile);
+                fileCount++;
+            }
+
+            return subData;
+        }
+
+        private static List<HashSet<T>> SplitRowList<T>(HashSet<T> list, int chunkSize)
+        {
+            var chunks = new List<HashSet<T>>();
+            if (list == null || chunkSize <= 0)
+                return chunks;
+
+            var array = list.ToArray();
+            for (int i = 0; i < array.Length; i += chunkSize)
+            {
+                var chunk = array.Skip(i).Take(chunkSize);
+                chunks.Add(new HashSet<T>(chunk));
+            }
+            return chunks;
+        }
     }
 }
diff --git a/Entities/Utilities/ExportUtilityBase.cs b/Entities/Utilities/ExportUtilityBase.cs
@@ -16,7 +16,7 @@ public class ExportUtilityBase
         protected readonly IConfiguration _config;
         protected readonly IExportDal _exportDal;
         protected int ExportMaxTablesPerFile { get; set; }
-        protected int MaxRowsPerFile { get; set; }
+        protected int MaxRowsPerTable { get; set; }
         protected decimal MaxRequestSize { get; set; }
         protected bool IsDevelopmentEnvironment { get; set; }
         public Guid ProcessId { get; set; }
@@ -37,7 +37,7 @@ public ExportUtilityBase(
         {
             _logger = logger;
             ExportMaxTablesPerFile = Convert.ToInt32(appSettings.ExportMaxTablesPerFile);
-            MaxRowsPerFile = Convert.ToInt32(appSettings.MaxRowsPerFile);
+            MaxRowsPerTable = Convert.ToInt32(appSettings.MaxRowsPerTable);
             ExportLookbackInDays = appSettings.ExportLookbackInDays;
             MaxRequestSize = Convert.ToDecimal(appSettings.MaximumRequestSize) / 1024 / 1024;
             ProcessId = processId;

Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,7 @@ public class DbCredentialsManager`
`10`	`10`	`private readonly Dictionary<string, DbCredentials> _dbCredentials = new Dictionary<string, DbCredentials>()`
`11`	`11`	`{`
`12`	`12`	`{`
`13`		`- "LocalizationTest", new DbCredentials() { Username = "LocalizationUser", Password = "@LocalizationTest#"}`
	`13`	`+ "LocalizationTest", new DbCredentials() { ServerName = "localhost", DatabaseName = "LocalizationTest", Username = "LocalizationUser", Password = "@LocalizationTest#"}`
`14`	`14`	`}`
`15`	`15`	`};`
`16`	`16`	`public DbCredentials GetDbCredentials(string connectionStringName)`
`@@ -21,6 +21,8 @@ public DbCredentials GetDbCredentials(string connectionStringName)`
`21`	`21`
`22`	`22`	`public class DbCredentials`
`23`	`23`	`{`
	`24`	`+ public string DatabaseName { get; set; }`
	`25`	`+ public string ServerName { get; set; }`
`24`	`26`	`public string Username { get; set; }`
`25`	`27`	`public string Password { get; set; }`
`26`	`28`	`}`
Original file line number	Diff line number	Diff line change
`@@ -14,7 +14,7 @@ public interface ISmartlingExportFileProcessor`
`14`	`14`	`string ExportType { get; set; }`
`15`	`15`	`ICacheManager CacheManager { get; set; }`
`16`	`16`	`Dictionary<string, List<DataSet>> DatabaseUpdatesForApp { get; set; }`
`17`		`- T ProcessExportData<T>(string exportLookbackInDays, int exportMaxTablesPerFile, int maxRowsPerFile, CancellationToken ct, Database db);`
	`17`	`+ T ProcessExportData<T>(string exportLookbackInDays, int exportMaxTablesPerFile, int maxRowsPerTable, CancellationToken ct, Database db);`
`18`	`18`	`MultipartFormDataContent GetCultureContent(SmartlingExportFile culturePackage, List<SmartlingLocaleId> localeIds, string callbackUrl = default(string));`
`19`	`19`	`}`
`20`	`20`	`}`
Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,7 @@ public SmartlingExportFileProcessor(`
`36`	`36`	`_smartlingConfiguration = smartlingConfig;`
`37`	`37`	`}`
`38`	`38`
`39`		`- public T ProcessExportData<T>(string exportLookbackInDays, int exportMaxTablesPerFile, int maxRowsPerFile, CancellationToken ct, Database db)`
	`39`	`+ public T ProcessExportData<T>(string exportLookbackInDays, int exportMaxTablesPerFile, int maxRowsPerTable, CancellationToken ct, Database db)`
`40`	`40`	`{`
`41`	`41`	`Dictionary<string, ExportTable> exportTables = new Dictionary<string, ExportTable>();`
`42`	`42`	`Dictionary<string, ExportTable> allTablesToExport = new Dictionary<string, ExportTable>();`