Skip to content

Commit 7c9a9c7

Browse files
AlexSanchez-bitAlexSanchez-bit
committed
fix[backend](initial-setup): improved filter insertion on system load
1 parent e36994b commit 7c9a9c7

6 files changed

Lines changed: 80 additions & 16 deletions

File tree

‎backend/src/main/java/com/park/utmstack/domain/correlation/config/UtmDataTypes.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,10 @@ public class UtmDataTypes implements Serializable {
4646
@Column(name = "system_owner", nullable = false)
4747
private Boolean systemOwner;
4848

49+
@ManyToOne(fetch = FetchType.LAZY)
50+
@JoinColumn(name = "module_id")
51+
private com.park.utmstack.domain.application_modules.UtmModule module;
52+
4953
@ManyToMany(fetch = FetchType.LAZY, cascade = {
5054
CascadeType.PERSIST,
5155
CascadeType.MERGE
@@ -121,4 +125,12 @@ public Boolean getSystemOwner() {
121125
public void setSystemOwner(Boolean systemOwner) {
122126
this.systemOwner = systemOwner;
123127
}
128+
129+
public com.park.utmstack.domain.application_modules.UtmModule getModule() {
130+
return module;
131+
}
132+
133+
public void setModule(com.park.utmstack.domain.application_modules.UtmModule module) {
134+
this.module = module;
135+
}
124136
}

‎backend/src/main/java/com/park/utmstack/repository/logstash_filter/UtmLogstashFilterRepository.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,12 @@ public interface UtmLogstashFilterRepository extends JpaRepository<UtmLogstashFi
2727

2828
Optional<UtmLogstashFilter> findOneByModuleName(String moduleName);
2929

30+
Optional<UtmLogstashFilter> findFirstByDatatypeDataType(String dataType);
31+
3032
Optional<UtmLogstashFilter> findFirstBySystemOwnerIsTrueOrderByIdDesc();
3133

34+
Optional<UtmLogstashFilter> findFirstByLogstashFilterAndSystemOwnerIsTrue(String logstashFilter);
35+
3236
@Query("select ulf from UtmLogstashFilter ulf where ulf.id in (:filterList) and ulf.systemOwner=false")
3337
List<UtmLogstashFilter> findAllByListOfId(@Param("filterList") List<Long> filterList);
3438

‎backend/src/main/java/com/park/utmstack/service/DefinitionSyncService.java‎

Lines changed: 41 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ public class DefinitionSyncService implements CommandLineRunner {
4646
private final UtmLogstashFilterService filterService;
4747

4848
@Override
49+
@Transactional
4950
public void run(String... args) {
5051
log.info("Starting definition sync from filesystem... ---");
5152
try {
@@ -69,23 +70,51 @@ private Set<String> syncFilters() {
6970
return foundModules;
7071
}
7172

73+
// Regex to extract the first dataType from the pipeline structure:
74+
// pipeline:
75+
// - dataTypes:
76+
// - value
77+
java.util.regex.Pattern dataTypePattern = java.util.regex.Pattern.compile(
78+
"pipeline:\\s*\\n\\s*-\\s*dataTypes:\\s*\\n\\s*-\\s*([^\\s\\n]+)",
79+
java.util.regex.Pattern.MULTILINE
80+
);
81+
7282
try (Stream<Path> paths = Files.walk(filtersPath)) {
7383
paths.filter(path -> Files.isRegularFile(path) && isYamlFile(path)).forEach(path -> {
74-
String rawName = getFileNameWithoutExtension(path);
75-
String moduleName = rawName.toUpperCase().replace("-", "_");
76-
foundModules.add(moduleName);
7784
try {
7885
String content = Files.readString(path);
86+
java.util.regex.Matcher matcher = dataTypePattern.matcher(content);
87+
if (!matcher.find()) {
88+
log.warn("Skipping filter file without dataType: {}", path);
89+
return;
90+
}
91+
92+
String dataTypeStr = matcher.group(1).trim().replace("\"", "").replace("'", "");
93+
log.info("found dataType: {}", dataTypeStr);
94+
95+
Optional<UtmDataTypes> dataTypeEntity = dataTypesRepository.findOneByDataType(dataTypeStr.toLowerCase());
96+
97+
String moduleName = null;
98+
if (dataTypeEntity.isPresent() && dataTypeEntity.get().getModule() != null) {
99+
moduleName = dataTypeEntity.get().getModule().getModuleName().toString();
100+
}
101+
102+
if(moduleName==null){
103+
log.error("module name for filter: {} with dataType: {} not found, ignoring...",path,dataTypeStr);
104+
return;
105+
}
79106

80-
Optional<UtmLogstashFilter> filterOpt = filterRepository.findOneByModuleName(moduleName);
107+
foundModules.add(moduleName);
108+
109+
Optional<UtmLogstashFilter> filterOpt = filterRepository.findFirstByLogstashFilterAndSystemOwnerIsTrue(content);
81110

82111
if (filterOpt.isPresent()) {
83112
UtmLogstashFilter filter = filterOpt.get();
84113
if (!content.equals(filter.getLogstashFilter())) {
85114
log.info("Updating existing filter for module: {}", moduleName);
86115
filter.setLogstashFilter(content);
87116
filter.setUpdatedAt(Instant.now());
88-
filterService.save(filter, true);
117+
filterService.save(filter);
89118
}
90119
} else {
91120
UtmLogstashFilter filter = new UtmLogstashFilter();
@@ -97,16 +126,18 @@ private Set<String> syncFilters() {
97126
filter.setActive(true);
98127
filter.setUpdatedAt(Instant.now());
99128

100-
// Try to find a matching data type
101-
Optional<UtmDataTypes> dataType = dataTypesRepository.findOneByDataType(moduleName.toLowerCase());
102-
if (dataType.isPresent()) {
103-
filter.setDatatype(dataType.get());
129+
130+
if (dataTypeEntity.isPresent()) {
131+
filter.setDatatype(dataTypeEntity.get());
104132
}
105133

106-
filterService.save(filter, true);
134+
filterService.save(filter);
135+
log.info("Creating filter from file {} for module: {} and dataType {}, filter: {}",path,moduleName, dataTypeStr,filter);
107136
}
108137
} catch (IOException e) {
109138
log.error("Error reading filter file {}: {}", path, e.getMessage());
139+
} catch (Exception e) {
140+
log.error("Error processing filter file {}: {}", path, e.getMessage());
110141
}
111142
});
112143
} catch (IOException e) {

‎backend/src/main/java/com/park/utmstack/service/logstash_filter/UtmLogstashFilterService.java‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,14 +35,9 @@ public UtmLogstashFilterService(UtmLogstashFilterRepository logstashFilterReposi
3535
* @return the persisted entity
3636
*/
3737
public UtmLogstashFilter save(UtmLogstashFilter logstashFilter) {
38-
return save(logstashFilter, false);
39-
}
40-
41-
public UtmLogstashFilter save(UtmLogstashFilter logstashFilter, boolean forcedSystemMode) {
4238
final String ctx = CLASSNAME + ".save";
4339
try {
4440
logstashFilter.setUpdatedAt(Instant.now());
45-
logstashFilter.setSystemOwner(forcedSystemMode);
4641
return logstashFilterRepository.save(logstashFilter);
4742
} catch (Exception e) {
4843
throw new RuntimeException(ctx + ": " + e.getMessage());
@@ -122,7 +117,7 @@ public List<UtmLogstashFilter> filtersByPipelineId(Long pipelineId) {
122117

123118
public Long getSystemSequenceNextValue() {
124119
return logstashFilterRepository.findFirstBySystemOwnerIsTrueOrderByIdDesc()
125-
.map(filter -> filter.getId() + 1)
120+
.map(filter -> (filter.getId() + 1)%1L)
126121
.orElse(1L);
127122
}
128123
}

‎backend/src/main/resources/config/liquibase/changelog/20260408001_add_module_id_to_utm_data_types.xml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<databaseChangeLog
3+
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5+
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
6+
7+
<changeSet id="20260408001" author="Alex">
8+
<addColumn tableName="utm_data_types">
9+
<column name="module_id" type="bigint">
10+
<constraints nullable="true"/>
11+
</column>
12+
</addColumn>
13+
14+
<addForeignKeyConstraint baseColumnNames="module_id"
15+
baseTableName="utm_data_types"
16+
constraintName="fk_utm_data_types_module_id"
17+
referencedColumnNames="id"
18+
referencedTableName="utm_module"/>
19+
</changeSet>
20+
</databaseChangeLog>

‎backend/src/main/resources/config/liquibase/master.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -585,4 +585,6 @@
585585

586586
<include file="/config/liquibase/changelog/20260403001_update_socai_group_config.xml" relativeToChangelogFile="false"/>
587587

588+
<include file="/config/liquibase/changelog/20260408001_add_module_id_to_utm_data_types.xml" relativeToChangelogFile="false"/>
589+
588590
</databaseChangeLog>

0 commit comments

Comments
 (0)