[{"createTime":1735734952000,"id":1,"img":"hwy_ms_500_252.jpeg","link":"https://activity.huaweicloud.com/cps.html?fromacct=261f35b6-af54-4511-a2ca-910fa15905d1&utm_source=V1g3MDY4NTY=&utm_medium=cps&utm_campaign=201905","name":"华为云秒杀","status":9,"txt":"华为云38元秒杀","type":1,"updateTime":1735747411000,"userId":3},{"createTime":1736173885000,"id":2,"img":"txy_480_300.png","link":"https://cloud.tencent.com/act/cps/redirect?redirect=1077&cps_key=edb15096bfff75effaaa8c8bb66138bd&from=console","name":"腾讯云秒杀","status":9,"txt":"腾讯云限量秒杀","type":1,"updateTime":1736173885000,"userId":3},{"createTime":1736177492000,"id":3,"img":"aly_251_140.png","link":"https://www.aliyun.com/minisite/goods?userCode=pwp8kmv3","memo":"","name":"阿里云","status":9,"txt":"阿里云2折起","type":1,"updateTime":1736177492000,"userId":3},{"createTime":1735660800000,"id":4,"img":"vultr_560_300.png","link":"https://www.vultr.com/?ref=9603742-8H","name":"Vultr","status":9,"txt":"Vultr送$100","type":1,"updateTime":1735660800000,"userId":3},{"createTime":1735660800000,"id":5,"img":"jdy_663_320.jpg","link":"https://3.cn/2ay1-e5t","name":"京东云","status":9,"txt":"京东云特惠专区","type":1,"updateTime":1735660800000,"userId":3},{"createTime":1735660800000,"id":6,"img":"new_ads.png","link":"https://www.iodraw.com/ads","name":"发布广告","status":9,"txt":"发布广告","type":1,"updateTime":1735660800000,"userId":3},{"createTime":1735660800000,"id":7,"img":"yun_910_50.png","link":"https://activity.huaweicloud.com/discount_area_v5/index.html?fromacct=261f35b6-af54-4511-a2ca-910fa15905d1&utm_source=aXhpYW95YW5nOA===&utm_medium=cps&utm_campaign=201905","name":"底部","status":9,"txt":"高性能云服务器2折起","type":2,"updateTime":1735660800000,"userId":3}]
<>前言
流式查询指的是查询成功后不是返回一个集合,而是返回一个迭代器,应用每次从迭代器取一条查询结果。流式查询的好处是能降低内存的使用。
如果没有使用流式查询,我们想要从数据库取1000万条记录而又没有足够的内存时,就不得不分页查询,而分页查询效率取决于表设计,如果设计得不好是,就无法高效的分页查询。因此流式查询是一个数据库访问框架必须具备的功能。
流式查询的过程当中,数据库连接是保持打开状态的,因此要注意的是:执行一个流式查询后,数据库访问框架就不负责关闭数据库连接了,需要应用在取完数据后自己关闭。
SpringBoot中应用
* UserMapper /** * 流式查询所有user * @return 返回的类型是游标类型 *
当查询百万级的数据的时候,使用游标可以节省内存的消耗,不需要一次性取出所有数据,可以进行逐条处理或逐条取出部分批量处理。 */ Cursor<User>
selectAll2(); //返回的是游标对象类型
* UserMapper.xml
在Mapper映射文件的标签中加入fetchSize=100
。因为mybatis默认是把所有数据全部查询出后返回,这样容易造成OOM问题(内存溢出),这时,我们设置一下fetchSize,设置每次查询出多少条数据,之后再执行next方法,读取下一批数据,这样每次查询出来一点,就处理一点,就不容易造成OOM问题了。
<!--加上fetchSize--> <select id="selectAll2" resultType="
com.age.batch.batchuser.entity.User" fetchSize="100"> select
id,username,age,sex,address from user</select>
* UserService /** * 流式查询所有user * @return */ Cursor<User> selectAdd2();
* UserServiceImpl /** * 流式查询所有user * @return */ @Override public Cursor<User>
selectAdd2() { return userMapper.selectAll2(); }
* controller @ApiOperation("流式查询所有user,测试查询时间") @GetMapping("/selectall2")
@Transactional
//注意要使用@Transactional注解来维持数据库连接,否则当recordMapper查询结束后数据库连接就会断开,Cursor就取不到数据了
public void selectAll2(){ Cursor<User> users = userService.selectAdd2(); users.
forEach(user -> System.out.println(user)); }
5000条数据,普通查询下的耗时为1366毫秒,流式查询下的耗时为76毫秒