xiaoming728

xiaoming728

Mysql分库分表

Mysql技术
95
2023-12-11
Mysql分库分表

引入依赖

    <!--shardingsphere-->
        <dependency>
            <groupId>org.apache.shardingsphere</groupId>
            <artifactId>shardingsphere-jdbc-core-spring-boot-starter</artifactId>
            <version>5.0.0-alpha</version>
        </dependency>
        <!--druid-->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>druid-spring-boot-starter</artifactId>
            <version>1.2.11</version>
        </dependency>

配置数据源

shardingsphere结合druid配置数据源，因为shardingsphere 5.X有common属性，所以一些公共的配置就写在了一块（最后会提供完整的配置文件）

## 数据源配置
spring:
    autoconfigure:
        exclude: com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceAutoConfigure
    datasource:
        type: com.alibaba.druid.pool.DruidDataSource
        druid:
            stat-view-servlet:
                enabled: true
                # 设置白名单，不填则允许所有访问
                allow:
                url-pattern: /druid/*
                loginUsername: admin
                loginPassword: 123456
            web-stat-filter:
                enabled: true
            filter:
                stat:
                    enabled: true
                    # 慢SQL记录
                    log-slow-sql: true
                    slow-sql-millis: 1000
                    merge-sql: true
                wall:
                    config:
                        multi-statement-allow: true
    shardingsphere:
        datasource:
            common:
                type: com.alibaba.druid.pool.DruidDataSource
                driver-class-name: com.mysql.cj.jdbc.Driver
                # 初始连接数
                initial-size: 6
                # 最小连接池数量
                min-idle: 3
                # 最大连接池数量
                maxActive: 20
                # 配置获取连接等待超时的时间
                maxWait: 60000
                # 配置间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒
                timeBetweenEvictionRunsMillis: 60000
                # 配置一个连接在池中最小生存的时间，单位是毫秒
                minEvictableIdleTimeMillis: 300000
                # 配置一个连接在池中最大生存的时间，单位是毫秒
                maxEvictableIdleTimeMillis: 900000
                #Oracle需要打开注释
                validationQuery: SELECT 1 FROM DUAL
                testWhileIdle: true
                testOnBorrow: false
                testOnReturn: false
                # 打开PSCache，并且指定每个连接上PSCache的大小
                poolPreparedStatements: true
                maxPoolPreparedStatementPerConnectionSize: 20
                # 配置监控统计拦截的filters，去掉后监控界面sql无法统计，'wall'用于防火墙
                filters: stat,wall,slf4j
                # 通过connectProperties属性来打开mergeSql功能；慢SQL记录
                connectionProperties: druid.stat.mergeSql\=true;druid.stat.slowSqlMillis\=5000
                wall:
                    multi-statement-allow: true
            #配置分库分表的数据源
            names: db0,db1,db2
            #配置数据库连接地址
            db0:
                url: jdbc:mysql://127.0.0.1:3306/ry_vue?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=false&serverTimezone=GMT%2B8
                username: root
                password: 123456
            db1:
                url: jdbc:mysql://127.0.0.1:3306/ry?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=false&serverTimezone=GMT%2B8
                username: root
                password: 123456
            db2:
                url: jdbc:mysql://127.0.0.1:3306/ry1?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=false&serverTimezone=GMT%2B8
                username: root
                password: 123456

druid配置类

去除druid底部广告

package com.ruoyi.framework.config;
 
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.sql.DataSource;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.web.servlet.FilterRegistrationBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder;
import com.alibaba.druid.spring.boot.autoconfigure.properties.DruidStatProperties;
import com.alibaba.druid.util.Utils;
import com.ruoyi.common.utils.spring.SpringUtils;
import com.ruoyi.framework.aspectj.lang.enums.DataSourceType;
import com.ruoyi.framework.datasource.DynamicDataSource;
 
/**
 * druid 配置多数据源
 *
 * @author ruoyi
 */
@Configuration
public class DruidConfig{
 
 
    /**
     * 去除监控页面底部的广告
     */
    @SuppressWarnings({ "rawtypes", "unchecked" })
    @Bean
    @ConditionalOnProperty(name = "spring.datasource.druid.stat-view-servlet.enabled", havingValue = "true")
    public FilterRegistrationBean removeDruidFilterRegistrationBean(DruidStatProperties properties)
    {
        // 获取web监控页面的参数
        DruidStatProperties.StatViewServlet config = properties.getStatViewServlet();
        // 提取common.js的配置路径
        String pattern = config.getUrlPattern() != null ? config.getUrlPattern() : "/druid/*";
        String commonJsPattern = pattern.replaceAll("\\*", "js/common.js");
        final String filePath = "support/http/resources/js/common.js";
        // 创建filter进行过滤
        Filter filter = new Filter()
        {
            @Override
            public void init(javax.servlet.FilterConfig filterConfig) throws ServletException
            {
            }
            @Override
            public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
                    throws IOException, ServletException
            {
                chain.doFilter(request, response);
                // 重置缓冲区，响应头不会被重置
                response.resetBuffer();
                // 获取common.js
                String text = Utils.readFromResource(filePath);
                // 正则替换banner, 除去底部的广告信息
                text = text.replaceAll("<a.*?banner\"></a><br/>", "");
                text = text.replaceAll("powered.*?shrek.wang</a>", "");
                response.getWriter().write(text);
            }
            @Override
            public void destroy()
            {
            }
        };
        FilterRegistrationBean registrationBean = new FilterRegistrationBean();
        registrationBean.setFilter(filter);
        registrationBean.addUrlPatterns(commonJsPattern);
        return registrationBean;
    }
}

兼容druid和shardingsphere

package com.ruoyi.framework.config;
 
import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceAutoConfigure;
import com.alibaba.druid.spring.boot.autoconfigure.properties.DruidStatProperties;
import com.alibaba.druid.spring.boot.autoconfigure.stat.DruidFilterConfiguration;
import com.alibaba.druid.spring.boot.autoconfigure.stat.DruidSpringAopConfiguration;
import com.alibaba.druid.spring.boot.autoconfigure.stat.DruidStatViewServletConfiguration;
import com.alibaba.druid.spring.boot.autoconfigure.stat.DruidWebStatFilterConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
 
/**
 * @authoer:majinzhong
 * @Date: 2022/11/11
 * @description:
 */
@Configuration
@ConditionalOnClass(DruidDataSourceAutoConfigure.class)
@EnableConfigurationProperties({DruidStatProperties.class})
@Import({
        DruidSpringAopConfiguration.class,
        DruidStatViewServletConfiguration.class,
        DruidWebStatFilterConfiguration.class,
        DruidFilterConfiguration.class})
public class DruidShardingJdbcDataSourceConfiguration {
}

读写分离

        rules:
            #配置读写分离
            replica-query:
                data-sources:
                    db:
                        # 读写分离类型，比如：Static，Dynamic，动态方式需要配合高可用功能，具体参考下方链接
                        # https://blog.csdn.net/ShardingSphere/article/details/123243843
                        type: Static
                        #主数据源名称
                        primary-data-source-name: master
                        #从数据源名称，多个用逗号分隔
                        replica-data-source-names: slave0,slave1
                        #负载均衡算法名称
                        load-balancer-name: round-robin
                        props:
                            # 注意，如果接口有事务，读写分离不生效，默认全部使用主库，为了保证数据一致性
                            write-data-source-name: master
                            read-data-source-names: slave0,slave1
                load-balancers:
                    round-robin:
                        #负载均衡算法配置，一共三种一种是 RANDOM（随机），一种是 ROUND_ROBIN（轮询），一种是 WEIGHT（权重）
                        type: ROUND_ROBIN
                        props:
                            #负载均衡算法属性配置
                            workId: 1
        props:
            #打印sql
            sql-show: true

配置分库分表

rules:
            sharding:
                #                default-data-source-name: db0
                # 分布式序列算法配置
                key-generators:
                    #　分布式序列算法（雪花算法：SNOWFLAKE, UUID:UUID(UUID没有props配置)）
                    snowflake:
                        type: SNOWFLAKE
                        props:
                            worker-id: 123
                # 配置user_info表
                tables:
                    foundation_place:
                        # 主键ＩＤ生成策略
                        key-generate-strategy:
                            #　分布式序列算法（雪花算法：SNOWFLAKE, UUID:UUID(UUID没有props配置)）
                            key-generator-name: snowflake
                            column: id
                        # 配置分表策略
                        actual-data-nodes: db$->{0..2}.foundation_place
#                        # 分库策略
                        database-strategy:
                            standard:
                                sharding-column: id
                                sharding-algorithm-name: database-inline
 
                #                        #　单分片键的标准分片
                #                        table-strategy:
                #                            standard:
                #                                sharding-column: id
                #                                sharding-algorithm-name: table-inline
                sharding-algorithms:
                    # 通过id取模的方式确定数据落在哪个库
                    database-inline:
                        type: INLINE
                        props:
                            algorithm-expression: db$->{id % 3}
        #                    # 通过id取模的方式确定数据落在哪个表
        #                    table-inline:
        #                        type: INLINE
        #                        props:
        #                            algorithm-expression: foundation_place
        #                enabled: true
        props:
            #打印sql
            sql-show: true

完整的配置文件

## 数据源配置
spring:
    autoconfigure:
        exclude: com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceAutoConfigure
    datasource:
        type: com.alibaba.druid.pool.DruidDataSource
        druid:
            stat-view-servlet:
                enabled: true
                # 设置白名单，不填则允许所有访问
                allow:
                url-pattern: /druid/*
                loginUsername: admin
                loginPassword: 123456
            web-stat-filter:
                enabled: true
            filter:
                stat:
                    enabled: true
                    # 慢SQL记录
                    log-slow-sql: true
                    slow-sql-millis: 1000
                    merge-sql: true
                wall:
                    config:
                        multi-statement-allow: true
    shardingsphere:
        datasource:
            common:
                type: com.alibaba.druid.pool.DruidDataSource
                driver-class-name: com.mysql.cj.jdbc.Driver
                # 初始连接数
                initial-size: 6
                # 最小连接池数量
                min-idle: 3
                # 最大连接池数量
                maxActive: 20
                # 配置获取连接等待超时的时间
                maxWait: 60000
                # 配置间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒
                timeBetweenEvictionRunsMillis: 60000
                # 配置一个连接在池中最小生存的时间，单位是毫秒
                minEvictableIdleTimeMillis: 300000
                # 配置一个连接在池中最大生存的时间，单位是毫秒
                maxEvictableIdleTimeMillis: 900000
                #Oracle需要打开注释
                validationQuery: SELECT 1 FROM DUAL
                testWhileIdle: true
                testOnBorrow: false
                testOnReturn: false
                # 打开PSCache，并且指定每个连接上PSCache的大小
                poolPreparedStatements: true
                maxPoolPreparedStatementPerConnectionSize: 20
                # 配置监控统计拦截的filters，去掉后监控界面sql无法统计，'wall'用于防火墙
                filters: stat,wall,slf4j
                # 通过connectProperties属性来打开mergeSql功能；慢SQL记录
                connectionProperties: druid.stat.mergeSql\=true;druid.stat.slowSqlMillis\=5000
                wall:
                    multi-statement-allow: true
            #配置分库分表的数据源
            names: db0,db1,db2
            #配置数据库连接地址
            db0:
                url: jdbc:mysql://127.0.0.1:3306/ry_vue?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=false&serverTimezone=GMT%2B8
                username: root
                password: 123456
            db1:
                url: jdbc:mysql://127.0.0.1:3306/ry?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=false&serverTimezone=GMT%2B8
                username: root
                password: 123456
            db2:
                url: jdbc:mysql://127.0.0.1:3306/ry1?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=false&serverTimezone=GMT%2B8
                username: root
                password: 123456
        rules:
            #配置读写分离
            replica-query:
                data-sources:
                    db:
                        # 读写分离类型，比如：Static，Dynamic，动态方式需要配合高可用功能，具体参考下方链接
                        # https://blog.csdn.net/ShardingSphere/article/details/123243843
                        type: Static
                        #主数据源名称
                        primary-data-source-name: master
                        #从数据源名称，多个用逗号分隔
                        replica-data-source-names: slave0,slave1
                        #负载均衡算法名称
                        load-balancer-name: round-robin
                        props:
                            # 注意，如果接口有事务，读写分离不生效，默认全部使用主库，为了保证数据一致性
                            write-data-source-name: master
                            read-data-source-names: slave0,slave1
                load-balancers:
                    round-robin:
                        #负载均衡算法配置，一共三种一种是 RANDOM（随机），一种是 ROUND_ROBIN（轮询），一种是 WEIGHT（权重）
                        type: ROUND_ROBIN
                        props:
                            #负载均衡算法属性配置
                            workId: 1
            sharding:
                #                default-data-source-name: db0
                # 分布式序列算法配置
                key-generators:
                    #　分布式序列算法（雪花算法：SNOWFLAKE, UUID:UUID(UUID没有props配置)）
                    snowflake:
                        type: SNOWFLAKE
                        props:
                            worker-id: 123
                # 配置user_info表
                tables:
                    foundation_place:
                        # 主键ＩＤ生成策略
                        key-generate-strategy:
                            #　分布式序列算法（雪花算法：SNOWFLAKE, UUID:UUID(UUID没有props配置)）
                            key-generator-name: snowflake
                            column: id
                        # 配置分表策略
                        actual-data-nodes: db$->{0..2}.foundation_place
#                        # 分库策略
                        database-strategy:
                            standard:
                                sharding-column: id
                                sharding-algorithm-name: database-inline
                #                        #　单分片键的标准分片
                #                        table-strategy:
                #                            standard:
                #                                sharding-column: id
                #                                sharding-algorithm-name: table-inline
                sharding-algorithms:
                    # 通过id取模的方式确定数据落在哪个库
                    database-inline:
                        type: INLINE
                        props:
                            algorithm-expression: db$->{id % 3}
        #                    # 通过id取模的方式确定数据落在哪个表
        #                    table-inline:
        #                        type: INLINE
        #                        props:
        #                            algorithm-expression: foundation_place
        #                enabled: true
        props:
            #打印sql
            sql-show: true

参考资料

shardingsphere 5.X结合druid实现读写分离，分库分表

Springboot整合shardingsphere

MySQL命令创建数据库与创建用户以及授权
Nginx 的五大应用场景