From 2cb85c39c6d55e6a85b28c483d1a162998bb236d Mon Sep 17 00:00:00 2001 From: kostyastruga Date: Sun, 16 Feb 2025 11:08:19 +0700 Subject: [PATCH 1/4] Change method for exist --- .../LocalResultStorageRepository.java | 12 ++++++++++++ .../repository/PaymentRepository.java | 2 ++ .../clickhouse/impl/FraudResultRepository.java | 11 +++++++++++ .../clickhouse/impl/PaymentRepositoryImpl.java | 18 ++++++++++++++++++ .../service/ShopManagementService.java | 2 +- ...uleCheckingServiceIntegrationTemplates.java | 2 +- .../repository/PaymentRepositoryTest.java | 7 +++++++ .../service/ShopManagementServiceTest.java | 5 +++-- 8 files changed, 55 insertions(+), 4 deletions(-) diff --git a/src/main/java/dev/vality/fraudbusters/fraud/localstorage/LocalResultStorageRepository.java b/src/main/java/dev/vality/fraudbusters/fraud/localstorage/LocalResultStorageRepository.java index f0a96d8d..d3f8d951 100644 --- a/src/main/java/dev/vality/fraudbusters/fraud/localstorage/LocalResultStorageRepository.java +++ b/src/main/java/dev/vality/fraudbusters/fraud/localstorage/LocalResultStorageRepository.java @@ -246,6 +246,18 @@ public Long sumOperationErrorWithGroupBy(String fieldName, .sum(); } + @Override + public Boolean isExistByField(String fieldName, Object value, Long from, Long to) { + List checkedPayments = localStorage.get(); + int count = (int) checkedPayments.stream() + .filter(checkedPayment -> checkedPayment.getEventTime() >= from + && checkedPayment.getEventTime() <= to + && paymentFieldValueFilter.filter(fieldName, value, checkedPayment)) + .count(); + log.debug("LocalResultStorageRepository countOperationByField: {}", count); + return count != 0; + } + private boolean filterByStatusAndFields( Long from, Long to, diff --git a/src/main/java/dev/vality/fraudbusters/repository/PaymentRepository.java b/src/main/java/dev/vality/fraudbusters/repository/PaymentRepository.java index d8fb390d..6086e9a8 100644 --- a/src/main/java/dev/vality/fraudbusters/repository/PaymentRepository.java +++ b/src/main/java/dev/vality/fraudbusters/repository/PaymentRepository.java @@ -57,4 +57,6 @@ Long sumOperationErrorWithGroupBy( Long to, List fieldModels); + + Boolean isExistByField(String fieldName, Object value, Long from, Long to); } diff --git a/src/main/java/dev/vality/fraudbusters/repository/clickhouse/impl/FraudResultRepository.java b/src/main/java/dev/vality/fraudbusters/repository/clickhouse/impl/FraudResultRepository.java index 1cffc6da..630089e0 100644 --- a/src/main/java/dev/vality/fraudbusters/repository/clickhouse/impl/FraudResultRepository.java +++ b/src/main/java/dev/vality/fraudbusters/repository/clickhouse/impl/FraudResultRepository.java @@ -273,4 +273,15 @@ public Long sumOperationErrorWithGroupBy(String fieldName, Object value, Long fr return sumOperationErrorWithGroupBy(fieldName, value, from, to, fieldModels, null); } + @Override + public Boolean isExistByField(String fieldName, Object value, Long from, Long to) { + return aggregationGeneralRepository.countOperationByField( + EventSource.FRAUD_EVENTS_UNIQUE.getTable(), + fieldName, + value, + from, + to + ) != 0; + } + } diff --git a/src/main/java/dev/vality/fraudbusters/repository/clickhouse/impl/PaymentRepositoryImpl.java b/src/main/java/dev/vality/fraudbusters/repository/clickhouse/impl/PaymentRepositoryImpl.java index 8f2d30e7..a6120941 100644 --- a/src/main/java/dev/vality/fraudbusters/repository/clickhouse/impl/PaymentRepositoryImpl.java +++ b/src/main/java/dev/vality/fraudbusters/repository/clickhouse/impl/PaymentRepositoryImpl.java @@ -332,4 +332,22 @@ public Long sumOperationErrorWithGroupBy(String fieldName, return jdbcTemplate.query(resultSql.toString(), params.toArray(), new SumExtractor()); } + @Override + public Boolean isExistByField(String fieldName, Object value, Long from, Long to) { + String sql = String.format(""" + select %1$s, count() as cnt + from %2$s + where timestamp >= ? + and timestamp <= ? + and eventTime >= ? + and eventTime <= ? + and %1$s = ? and status in (?, ?, ?) + group by %1$s + limit 1""", fieldName, TABLE); + List params = + AggregationUtil.generateStatusesParams(from, to, value, AggregationUtil.getFinalStatuses()); + log.debug("AggregationGeneralRepositoryImpl countOperationByField sql: {} params: {}", sql, params); + return jdbcTemplate.query(sql, params.toArray(), new CountExtractor()) != 0; + } + } diff --git a/src/main/java/dev/vality/fraudbusters/service/ShopManagementService.java b/src/main/java/dev/vality/fraudbusters/service/ShopManagementService.java index 291b3076..a481af3b 100644 --- a/src/main/java/dev/vality/fraudbusters/service/ShopManagementService.java +++ b/src/main/java/dev/vality/fraudbusters/service/ShopManagementService.java @@ -28,7 +28,7 @@ public boolean isNewShop(String partyId, String shopId) { } Long to = Instant.now().toEpochMilli(); Long from = Instant.now().minus(properties.getCountToCheckDays(), ChronoUnit.DAYS).toEpochMilli(); - return repository.countOperationByField("shopId", shopId, from, to) == 0; + return repository.isExistByField("shopId", shopId, from, to); } private boolean hasReferenceInPools(String partyId, String shopId) { diff --git a/src/test/java/dev/vality/fraudbusters/constants/RuleCheckingServiceIntegrationTemplates.java b/src/test/java/dev/vality/fraudbusters/constants/RuleCheckingServiceIntegrationTemplates.java index 99116f1f..ae44b34a 100644 --- a/src/test/java/dev/vality/fraudbusters/constants/RuleCheckingServiceIntegrationTemplates.java +++ b/src/test/java/dev/vality/fraudbusters/constants/RuleCheckingServiceIntegrationTemplates.java @@ -22,7 +22,7 @@ public class RuleCheckingServiceIntegrationTemplates { public static final String TEMPLATE = "rule: amount() > 5 -> accept;"; public static final String TEMPLATE_PARTY = "rule: amount() > 60 -> accept;"; public static final String TEMPLATE_SHOP = "rule: amount() > 55 -> accept;"; - public static final String PREVIOUS_TEMPLATE_PARTY = "rule: amount() > 200 -> accept;"; + public static final String PREVIOUS_TEMPLATE_PARTY = "rule: rand(40) > 20 -> trust;"; public static final String PREVIOUS_TEMPLATE_SHOP = "rule: amount() > 100 -> accept;"; } diff --git a/src/test/java/dev/vality/fraudbusters/repository/PaymentRepositoryTest.java b/src/test/java/dev/vality/fraudbusters/repository/PaymentRepositoryTest.java index e31a9212..6136a3c6 100644 --- a/src/test/java/dev/vality/fraudbusters/repository/PaymentRepositoryTest.java +++ b/src/test/java/dev/vality/fraudbusters/repository/PaymentRepositoryTest.java @@ -35,6 +35,7 @@ import static dev.vality.fraudbusters.extension.ClickHouseContainerExtension.CLICKHOUSE_CONTAINER; import static dev.vality.fraudbusters.util.BeanUtil.*; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; @Slf4j @ActiveProfiles("full-prod") @@ -77,6 +78,12 @@ public void countOperationByPhoneTest() throws SQLException { assertEquals(1, count); } + @Test + public void isExistTest() throws SQLException { + Boolean isExist = paymentRepository.isExistByField(EventField.phone.name(), PHONE, FROM, TO); + assertTrue(isExist); + } + @Test public void countOperationByEmailTestWithGroupBy() throws SQLException { PaymentModel paymentModel = createFraudModelSecond(); diff --git a/src/test/java/dev/vality/fraudbusters/service/ShopManagementServiceTest.java b/src/test/java/dev/vality/fraudbusters/service/ShopManagementServiceTest.java index 98abd4f7..e50644c8 100644 --- a/src/test/java/dev/vality/fraudbusters/service/ShopManagementServiceTest.java +++ b/src/test/java/dev/vality/fraudbusters/service/ShopManagementServiceTest.java @@ -68,15 +68,16 @@ public void testCreateDefaultReference() { @Test public void testIsNewShop() { - when(fraudResultRepository.countOperationByField(anyString(), anyString(), anyLong(), anyLong())).thenReturn(0); + when(fraudResultRepository.isExistByField(anyString(), anyString(), anyLong(), anyLong())).thenReturn(false); shopManagementService.isNewShop("partyId", "s1"); - verify(fraudResultRepository).countOperationByField(anyString(), anyString(), anyLong(), anyLong()); + verify(fraudResultRepository).isExistByField(anyString(), anyString(), anyLong(), anyLong()); referencePoolImpl.add("partyId", "test"); boolean newShop = shopManagementService.isNewShop("partyId", "s1"); assertFalse(newShop); referencePoolImpl.remove("partyId"); + when(fraudResultRepository.isExistByField(anyString(), anyString(), anyLong(), anyLong())).thenReturn(true); newShop = shopManagementService.isNewShop("partyId", "s1"); assertTrue(newShop); } From c1a45690ac5307c2d03b1c067b7e0e6663537440 Mon Sep 17 00:00:00 2001 From: kostyastruga Date: Sun, 16 Feb 2025 11:14:08 +0700 Subject: [PATCH 2/4] Change method for exist --- .../clickhouse/impl/PaymentRepositoryImpl.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/dev/vality/fraudbusters/repository/clickhouse/impl/PaymentRepositoryImpl.java b/src/main/java/dev/vality/fraudbusters/repository/clickhouse/impl/PaymentRepositoryImpl.java index a6120941..a332728c 100644 --- a/src/main/java/dev/vality/fraudbusters/repository/clickhouse/impl/PaymentRepositoryImpl.java +++ b/src/main/java/dev/vality/fraudbusters/repository/clickhouse/impl/PaymentRepositoryImpl.java @@ -335,18 +335,16 @@ public Long sumOperationErrorWithGroupBy(String fieldName, @Override public Boolean isExistByField(String fieldName, Object value, Long from, Long to) { String sql = String.format(""" - select %1$s, count() as cnt + select 1 as cnt from %2$s where timestamp >= ? and timestamp <= ? and eventTime >= ? and eventTime <= ? - and %1$s = ? and status in (?, ?, ?) - group by %1$s + and %1$s = ? limit 1""", fieldName, TABLE); - List params = - AggregationUtil.generateStatusesParams(from, to, value, AggregationUtil.getFinalStatuses()); - log.debug("AggregationGeneralRepositoryImpl countOperationByField sql: {} params: {}", sql, params); + List params = AggregationUtil.generateParams(from, to, value); + log.debug("AggregationGeneralRepositoryImpl isExistByField sql: {} params: {}", sql, params); return jdbcTemplate.query(sql, params.toArray(), new CountExtractor()) != 0; } From 6ef6c64ac43a7843d75314c19ebd4d8c923bb828 Mon Sep 17 00:00:00 2001 From: kostyastruga Date: Sun, 16 Feb 2025 12:09:16 +0700 Subject: [PATCH 3/4] Change method for exist --- .../constants/RuleCheckingServiceIntegrationTemplates.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/dev/vality/fraudbusters/constants/RuleCheckingServiceIntegrationTemplates.java b/src/test/java/dev/vality/fraudbusters/constants/RuleCheckingServiceIntegrationTemplates.java index ae44b34a..99116f1f 100644 --- a/src/test/java/dev/vality/fraudbusters/constants/RuleCheckingServiceIntegrationTemplates.java +++ b/src/test/java/dev/vality/fraudbusters/constants/RuleCheckingServiceIntegrationTemplates.java @@ -22,7 +22,7 @@ public class RuleCheckingServiceIntegrationTemplates { public static final String TEMPLATE = "rule: amount() > 5 -> accept;"; public static final String TEMPLATE_PARTY = "rule: amount() > 60 -> accept;"; public static final String TEMPLATE_SHOP = "rule: amount() > 55 -> accept;"; - public static final String PREVIOUS_TEMPLATE_PARTY = "rule: rand(40) > 20 -> trust;"; + public static final String PREVIOUS_TEMPLATE_PARTY = "rule: amount() > 200 -> accept;"; public static final String PREVIOUS_TEMPLATE_SHOP = "rule: amount() > 100 -> accept;"; } From 442b3eaab5ef8eef9a7dc2be53d66bc5d8a1ef6d Mon Sep 17 00:00:00 2001 From: kostyastruga Date: Mon, 17 Feb 2025 15:02:53 +0700 Subject: [PATCH 4/4] Change method for exist --- .../dev/vality/fraudbusters/service/ShopManagementService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/dev/vality/fraudbusters/service/ShopManagementService.java b/src/main/java/dev/vality/fraudbusters/service/ShopManagementService.java index a481af3b..0b117469 100644 --- a/src/main/java/dev/vality/fraudbusters/service/ShopManagementService.java +++ b/src/main/java/dev/vality/fraudbusters/service/ShopManagementService.java @@ -6,6 +6,7 @@ import dev.vality.fraudbusters.util.ReferenceKeyGenerator; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import java.time.Instant; @@ -22,6 +23,7 @@ public class ShopManagementService { private final Pool referencePoolImpl; private final Pool groupReferencePoolImpl; + @Cacheable(value = "isNewShop") public boolean isNewShop(String partyId, String shopId) { if (hasReferenceInPools(partyId, shopId)) { return false;