Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@ on:

jobs:
build:
uses: valitydev/java-workflow/.github/workflows/maven-service-build.yml@v1
uses: valitydev/java-workflow/.github/workflows/maven-service-build.yml@v3
2 changes: 1 addition & 1 deletion .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:

jobs:
deploy:
uses: valitydev/java-workflow/.github/workflows/maven-service-deploy.yml@v1
uses: valitydev/java-workflow/.github/workflows/maven-service-deploy.yml@v3
secrets:
github-token: ${{ secrets.GITHUB_TOKEN }}
mm-webhook-url: ${{ secrets.MATTERMOST_WEBHOOK_URL }}
33 changes: 16 additions & 17 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<parent>
<groupId>dev.vality</groupId>
<artifactId>service-parent-pom</artifactId>
<version>1.0.19</version>
<version>3.1.1</version>
</parent>

<artifactId>binbase</artifactId>
Expand All @@ -29,8 +29,8 @@
<db.user>postgres</db.user>
<db.password>postgres</db.password>
<db.schema>binbase</db.schema>
<damsel.version>1.568-9362c08</damsel.version>
<testcontainers.annotations.version>1.4.3</testcontainers.annotations.version>
<damsel.version>1.655-5ca4f4a</damsel.version>
<testcontainers.annotations.version>3.1.1</testcontainers.annotations.version>
</properties>

<dependencies>
Expand All @@ -52,7 +52,7 @@
<dependency>
<groupId>dev.vality</groupId>
<artifactId>cds-proto</artifactId>
<version>1.66-01353ce</version>
<version>1.70-5227377</version>
</dependency>

<!--spring -->
Expand All @@ -78,6 +78,10 @@
<groupId>org.flywaydb</groupId>
<artifactId>flyway-core</artifactId>
</dependency>
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-database-postgresql</artifactId>
</dependency>
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
Expand All @@ -95,25 +99,20 @@
<version>17.0.0</version>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.0</version>
<groupId>jakarta.xml.bind</groupId>
<artifactId>jakarta.xml.bind-api</artifactId>
<version>3.0.1</version>
</dependency>
<dependency>
<groupId>org.glassfish.jaxb</groupId>
<artifactId>jaxb-runtime</artifactId>
<version>2.3.0</version>
<version>3.0.2</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>javax.activation</groupId>
<artifactId>javax.activation-api</artifactId>
<version>1.2.0</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>1.3.2</version>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
Expand Down Expand Up @@ -150,9 +149,9 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>javax.xml.ws</groupId>
<artifactId>jaxws-api</artifactId>
<version>2.3.0</version>
<groupId>jakarta.xml.ws</groupId>
<artifactId>jakarta.xml.ws-api</artifactId>
<version>3.0.0</version>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import lombok.Data;
import lombok.NoArgsConstructor;

import javax.xml.bind.annotation.XmlRootElement;
import jakarta.xml.bind.annotation.XmlRootElement;

@Data
@NoArgsConstructor
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ public void beforeChunk(ChunkContext context) {

@Override
public void afterChunk(ChunkContext context) {
int readCount = context.getStepContext().getStepExecution().getReadCount();
long readCount = context.getStepContext().getStepExecution().getReadCount();
if (readCount > 0 && readCount % loggingInterval == 0) {
log.info("{} rows processed", readCount);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@
import dev.vality.binbase.domain.BinData;
import dev.vality.binbase.service.BinbaseService;
import lombok.RequiredArgsConstructor;
import org.springframework.batch.item.Chunk;
import org.springframework.batch.item.ItemWriter;
import org.springframework.stereotype.Component;

import java.util.List;
import java.util.Map;

@Component
Expand All @@ -17,7 +17,7 @@ public class BinRangeWriter implements ItemWriter<Map.Entry<BinData, Range<Long>
private final BinbaseService binbaseService;

@Override
public void write(List<? extends Map.Entry<BinData, Range<Long>>> binDataRanges) throws Exception {
public void write(Chunk<? extends Map.Entry<BinData, Range<Long>>> binDataRanges) throws Exception {
binDataRanges.forEach(binDataRange -> binbaseService.saveRange(binDataRange.getKey(), binDataRange.getValue()));
}

Expand Down
26 changes: 12 additions & 14 deletions src/main/java/dev/vality/binbase/config/BatchConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
import lombok.RequiredArgsConstructor;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.job.builder.JobBuilder;
import org.springframework.batch.core.repository.JobRepository;
import org.springframework.batch.core.step.builder.StepBuilder;
import org.springframework.batch.item.file.FlatFileItemReader;
import org.springframework.batch.item.file.MultiResourceItemReader;
import org.springframework.batch.item.file.builder.MultiResourceItemReaderBuilder;
Expand All @@ -29,17 +29,15 @@
import org.springframework.batch.item.support.builder.ClassifierCompositeItemProcessorBuilder;
import org.springframework.batch.item.xml.StaxEventItemReader;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.Resource;
import org.springframework.oxm.jaxb.Jaxb2Marshaller;
import org.springframework.transaction.PlatformTransactionManager;

import java.util.Map;

@Configuration
@EnableBatchProcessing
@EnableAutoConfiguration
@RequiredArgsConstructor
public class BatchConfig {

Expand All @@ -49,8 +47,8 @@ public class BatchConfig {
"isonumber", "url", "phone", "bin_length", "affiliation", "mark"
};

private final JobBuilderFactory jobBuilderFactory;
private final StepBuilderFactory stepBuilderFactory;
private final JobRepository jobRepository;
private final PlatformTransactionManager transactionManager;
private final BinbaseService binbaseService;

@Value("${batch.strict_mode}")
Expand Down Expand Up @@ -115,9 +113,8 @@ public FlatFileItemReader<BinBaseCsvData> buildBinBasePsbCsvReader() {

@Bean
public Job binBaseJob(Step step) {
return jobBuilderFactory.get("binBaseJob")
.flow(step)
.end()
return new JobBuilder("binBaseJob", jobRepository)
.start(step)
.build();
}

Expand All @@ -128,10 +125,11 @@ public ClassifierCompositeItemProcessor compositeProcessor() {
.build();
}


@Bean
public Step step(MultiResourceItemReader multiResourceItemReader) {
return stepBuilderFactory.get("binBaseStep")
.<Resource, Map.Entry<BinData, Range<Long>>>chunk(1000)
public Step step(MultiResourceItemReader<BinData> multiResourceItemReader) {
return new StepBuilder("binBaseStep", jobRepository)
.<BinData, Map.Entry<BinData, Range<Long>>>chunk(1000, transactionManager)
.reader(multiResourceItemReader)
.processor(compositeProcessor())
.writer(new BinRangeWriter(binbaseService))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
import dev.vality.woody.thrift.impl.http.THServiceBuilder;
import org.springframework.beans.factory.annotation.Autowired;

import javax.servlet.*;
import javax.servlet.annotation.WebServlet;
import jakarta.servlet.*;
import jakarta.servlet.annotation.WebServlet;
import java.io.IOException;

@WebServlet("/v1/binbase")
Expand Down
21 changes: 11 additions & 10 deletions src/main/resources/application.yml
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
server:
port: '@server.port@'
port: ${server.port}

spring:
application:
name: '@name@'
description: '@description@'
name: ${name}
description: ${description}
datasource:
type: com.zaxxer.hikari.HikariDataSource
driver-class-name: org.postgresql.Driver
url: '@db.url@'
username: '@db.user@'
password: '@db.password@'
url: ${db.url}
username: ${db.user}
password: ${db.password}
batch:
jdbc:
initialize-schema: always
Expand All @@ -22,11 +22,8 @@ batch:

management:
server:
port: '@management.port@'
port: ${management.port}
metrics:
export:
prometheus:
enabled: false
tags:
application: wb-list-manager
endpoint:
Expand All @@ -40,6 +37,10 @@ management:
web:
exposure:
include: health,info,prometheus
prometheus:
metrics:
export:
enabled: false

client:
cds:
Expand Down
12 changes: 10 additions & 2 deletions src/test/java/dev/vality/binbase/batch/BatchUploadTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,12 @@

import dev.vality.binbase.config.PostgresqlTest;
import dev.vality.binbase.domain.CountryCode;
import dev.vality.damsel.binbase.*;
import dev.vality.damsel.binbase.BinNotFound;
import dev.vality.damsel.binbase.BinbaseSrv;
import dev.vality.damsel.binbase.CardType;
import dev.vality.damsel.binbase.Last;
import dev.vality.damsel.binbase.Reference;
import dev.vality.damsel.binbase.ResponseData;
import dev.vality.woody.thrift.impl.http.THSpawnClientBuilder;
import org.apache.thrift.TException;
import org.junit.jupiter.api.BeforeEach;
Expand All @@ -14,7 +19,10 @@
import java.net.URI;
import java.net.URISyntaxException;

import static org.junit.jupiter.api.Assertions.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@

@Target({ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
@PostgresqlTestcontainerSingleton
@PostgresqlTestcontainerSingleton(truncateTables = false)
public @interface PostgresqlTest {
}
Loading