Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 9e41813

Browse files
Spring Batch读取数据
1 parent 82c9fc3 commit 9e41813

17 files changed

Lines changed: 668 additions & 0 deletions

‎68.spring-batch-itemreader/pom.xml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
4+
<modelVersion>4.0.0</modelVersion>
5+
<parent>
6+
<groupId>org.springframework.boot</groupId>
7+
<artifactId>spring-boot-starter-parent</artifactId>
8+
<version>2.2.5.RELEASE</version>
9+
<relativePath/> <!-- lookup parent from repository -->
10+
</parent>
11+
<groupId>cc.mrbird</groupId>
12+
<artifactId>spring-batch-itemreader</artifactId>
13+
<version>0.0.1-SNAPSHOT</version>
14+
<name>spring-batch-itemreader</name>
15+
<description>Demo project for Spring Boot</description>
16+
17+
<properties>
18+
<java.version>1.8</java.version>
19+
</properties>
20+
21+
<dependencies>
22+
<dependency>
23+
<groupId>org.springframework.boot</groupId>
24+
<artifactId>spring-boot-starter-batch</artifactId>
25+
</dependency>
26+
27+
<dependency>
28+
<groupId>mysql</groupId>
29+
<artifactId>mysql-connector-java</artifactId>
30+
</dependency>
31+
<dependency>
32+
<groupId>org.springframework.boot</groupId>
33+
<artifactId>spring-boot-starter-jdbc</artifactId>
34+
</dependency>
35+
36+
<dependency>
37+
<groupId>org.springframework</groupId>
38+
<artifactId>spring-oxm</artifactId>
39+
</dependency>
40+
<dependency>
41+
<groupId>com.thoughtworks.xstream</groupId>
42+
<artifactId>xstream</artifactId>
43+
<version>1.4.11.1</version>
44+
</dependency>
45+
</dependencies>
46+
47+
<build>
48+
<plugins>
49+
<plugin>
50+
<groupId>org.springframework.boot</groupId>
51+
<artifactId>spring-boot-maven-plugin</artifactId>
52+
</plugin>
53+
</plugins>
54+
</build>
55+
56+
</project>
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package cc.mrbird.batch;
2+
3+
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
4+
import org.springframework.boot.SpringApplication;
5+
import org.springframework.boot.autoconfigure.SpringBootApplication;
6+
7+
@SpringBootApplication
8+
@EnableBatchProcessing
9+
public class SpringBatchItemreaderApplication {
10+
11+
public static void main(String[] args) {
12+
SpringApplication.run(SpringBatchItemreaderApplication.class, args);
13+
}
14+
15+
}
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package cc.mrbird.batch.entity;
2+
3+
/**
4+
* @author MrBird
5+
*/
6+
public class TestData {
7+
private int id;
8+
private String field1;
9+
private String field2;
10+
private String field3;
11+
12+
public int getId() {
13+
return id;
14+
}
15+
16+
public void setId(int id) {
17+
this.id = id;
18+
}
19+
20+
public String getField1() {
21+
return field1;
22+
}
23+
24+
public void setField1(String field1) {
25+
this.field1 = field1;
26+
}
27+
28+
public String getField2() {
29+
return field2;
30+
}
31+
32+
public void setField2(String field2) {
33+
this.field2 = field2;
34+
}
35+
36+
public String getField3() {
37+
return field3;
38+
}
39+
40+
public void setField3(String field3) {
41+
this.field3 = field3;
42+
}
43+
44+
@Override
45+
public String toString() {
46+
return "TestData{" +
47+
"id=" + id +
48+
", field1='" + field1 + '\'' +
49+
", field2='" + field2 + '\'' +
50+
", field3='" + field3 + '\'' +
51+
'}';
52+
}
53+
}
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package cc.mrbird.batch.job;
2+
3+
import cc.mrbird.batch.entity.TestData;
4+
import org.springframework.batch.core.Job;
5+
import org.springframework.batch.core.Step;
6+
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
7+
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
8+
import org.springframework.batch.item.ItemReader;
9+
import org.springframework.batch.item.database.JdbcPagingItemReader;
10+
import org.springframework.batch.item.database.Order;
11+
import org.springframework.batch.item.database.support.MySqlPagingQueryProvider;
12+
import org.springframework.beans.factory.annotation.Autowired;
13+
import org.springframework.context.annotation.Bean;
14+
import org.springframework.stereotype.Component;
15+
16+
import javax.sql.DataSource;
17+
import java.util.HashMap;
18+
import java.util.Map;
19+
20+
/**
21+
* @author MrBird
22+
*/
23+
@Component
24+
public class DataSourceItemReaderDemo {
25+
26+
@Autowired
27+
private JobBuilderFactory jobBuilderFactory;
28+
@Autowired
29+
private StepBuilderFactory stepBuilderFactory;
30+
// 注入数据源
31+
@Autowired
32+
private DataSource dataSource;
33+
34+
@Bean
35+
public Job dataSourceItemReaderJob() throws Exception {
36+
return jobBuilderFactory.get("dataSourceItemReaderJob")
37+
.start(step())
38+
.build();
39+
}
40+
41+
private Step step() throws Exception {
42+
return stepBuilderFactory.get("step")
43+
.<TestData, TestData>chunk(2)
44+
.reader(dataSourceItemReader())
45+
.writer(list -> list.forEach(System.out::println))
46+
.build();
47+
}
48+
49+
private ItemReader<TestData> dataSourceItemReader() throws Exception {
50+
JdbcPagingItemReader<TestData> reader = new JdbcPagingItemReader<>();
51+
reader.setDataSource(dataSource); // 设置数据源
52+
reader.setFetchSize(5); // 每次取多少条记录
53+
reader.setPageSize(5); // 设置每页数据量
54+
55+
// 指定sql查询语句 select id,field1,field2,field3 from TEST
56+
MySqlPagingQueryProvider provider = new MySqlPagingQueryProvider();
57+
provider.setSelectClause("id,field1,field2,field3"); //设置查询字段
58+
provider.setFromClause("from TEST"); // 设置从哪张表查询
59+
60+
// 将读取到的数据转换为TestData对象
61+
reader.setRowMapper((resultSet, rowNum) -> {
62+
TestData data = new TestData();
63+
data.setId(resultSet.getInt(1));
64+
data.setField1(resultSet.getString(2)); // 读取第一个字段,类型为String
65+
data.setField2(resultSet.getString(3));
66+
data.setField3(resultSet.getString(4));
67+
return data;
68+
});
69+
70+
Map<String, Order> sort = new HashMap<>(1);
71+
sort.put("id", Order.ASCENDING);
72+
provider.setSortKeys(sort); // 设置排序,通过id 升序
73+
74+
reader.setQueryProvider(provider);
75+
76+
// 设置namedParameterJdbcTemplate等属性
77+
reader.afterPropertiesSet();
78+
return reader;
79+
}
80+
}
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
package cc.mrbird.batch.job;
2+
3+
import cc.mrbird.batch.entity.TestData;
4+
import org.springframework.batch.core.Job;
5+
import org.springframework.batch.core.Step;
6+
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
7+
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
8+
import org.springframework.batch.item.ItemReader;
9+
import org.springframework.batch.item.file.FlatFileItemReader;
10+
import org.springframework.batch.item.file.mapping.DefaultLineMapper;
11+
import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
12+
import org.springframework.beans.factory.annotation.Autowired;
13+
import org.springframework.context.annotation.Bean;
14+
import org.springframework.core.io.ClassPathResource;
15+
import org.springframework.stereotype.Component;
16+
17+
/**
18+
* @author MrBird
19+
*/
20+
@Component
21+
public class FileItemReaderDemo {
22+
// 任务创建工厂
23+
@Autowired
24+
private JobBuilderFactory jobBuilderFactory;
25+
// 步骤创建工厂
26+
@Autowired
27+
private StepBuilderFactory stepBuilderFactory;
28+
29+
@Bean
30+
public Job fileItemReaderJob() {
31+
return jobBuilderFactory.get("fileItemReaderJob")
32+
.start(step())
33+
.build();
34+
}
35+
36+
private Step step() {
37+
return stepBuilderFactory.get("step")
38+
.<TestData, TestData>chunk(2)
39+
.reader(fileItemReader())
40+
.writer(list -> list.forEach(System.out::println))
41+
.build();
42+
}
43+
44+
private ItemReader<TestData> fileItemReader() {
45+
FlatFileItemReader<TestData> reader = new FlatFileItemReader<>();
46+
reader.setResource(new ClassPathResource("file")); // 设置文件资源地址
47+
reader.setLinesToSkip(1); // 忽略第一行
48+
49+
// AbstractLineTokenizer的三个实现类之一,以固定分隔符处理行数据读取,
50+
// 使用默认构造器的时候,使用逗号作为分隔符,也可以通过有参构造器来指定分隔符
51+
DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
52+
53+
// 设置属性名,类似于表头
54+
tokenizer.setNames("id", "field1", "field2", "field3");
55+
// 将每行数据转换为TestData对象
56+
DefaultLineMapper<TestData> mapper = new DefaultLineMapper<>();
57+
mapper.setLineTokenizer(tokenizer);
58+
// 设置映射方式
59+
mapper.setFieldSetMapper(fieldSet -> {
60+
TestData data = new TestData();
61+
data.setId(fieldSet.readInt("id"));
62+
data.setField1(fieldSet.readString("field1"));
63+
data.setField2(fieldSet.readString("field2"));
64+
data.setField3(fieldSet.readString("field3"));
65+
return data;
66+
});
67+
68+
reader.setLineMapper(mapper);
69+
return reader;
70+
}
71+
}
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package cc.mrbird.batch.job;
2+
3+
import cc.mrbird.batch.entity.TestData;
4+
import org.springframework.batch.core.Job;
5+
import org.springframework.batch.core.Step;
6+
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
7+
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
8+
import org.springframework.batch.item.ItemReader;
9+
import org.springframework.batch.item.json.JacksonJsonObjectReader;
10+
import org.springframework.batch.item.json.JsonItemReader;
11+
import org.springframework.beans.factory.annotation.Autowired;
12+
import org.springframework.context.annotation.Bean;
13+
import org.springframework.core.io.ClassPathResource;
14+
import org.springframework.stereotype.Component;
15+
16+
/**
17+
* @author MrBird
18+
*/
19+
@Component
20+
public class JSONFileItemReaderDemo {
21+
22+
@Autowired
23+
private JobBuilderFactory jobBuilderFactory;
24+
@Autowired
25+
private StepBuilderFactory stepBuilderFactory;
26+
27+
@Bean
28+
public Job jsonFileItemReaderJob() {
29+
return jobBuilderFactory.get("jsonFileItemReaderJob")
30+
.start(step())
31+
.build();
32+
}
33+
34+
private Step step() {
35+
return stepBuilderFactory.get("step")
36+
.<TestData, TestData>chunk(2)
37+
.reader(jsonItemReader())
38+
.writer(list -> list.forEach(System.out::println))
39+
.build();
40+
}
41+
42+
private ItemReader<TestData> jsonItemReader() {
43+
// 设置json文件地址
44+
ClassPathResource resource = new ClassPathResource("file.json");
45+
// 设置json文件转换的目标对象类型
46+
JacksonJsonObjectReader<TestData> jacksonJsonObjectReader = new JacksonJsonObjectReader<>(TestData.class);
47+
JsonItemReader<TestData> reader = new JsonItemReader<>(resource, jacksonJsonObjectReader);
48+
// 给reader设置一个别名
49+
reader.setName("testDataJsonItemReader");
50+
return reader;
51+
}
52+
}

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /