The Linux system's buffers are set to low for high speed transfers.
The important settings below are:
Increasing TCP buffers:
- net.core.rmem_max=268431360
- net.core.wmem_max=134215680
- net.ipv4.tcp_rmem=4096 262144 268431360
- net.ipv4.tcp_wmem=4096 131072 134215680
Setting congestion control:
- net.ipv4.tcp_congestion_control=htcp
The below should be added to the bottom of sysctl.conf
/etc/sysctl.conf
# Science DMZ PERFORMANCE TUNING
net.ipv4.conf.all.accept_source_route=0
net.ipv4.conf.default.send_redirects=0
net.ipv4.conf.all.send_redirects=0
net.ipv4.icmp_echo_ignore_broadcasts=1
dev.raid.speed_limit_max=1410065407
net.ipv4.ipfrag_high_thresh=4194303
net.ipv4.ipfrag_low_thresh=1048575
net.core.rmem_max=268431360
net.core.wmem_max=134215680
net.core.rmem_default=262144
net.core.wmem_default=131072
net.core.somaxconn=16384
net.ipv4.tcp_rmem=4096 262144 268431360
net.ipv4.tcp_wmem=4096 131072 134215680
net.ipv4.tcp_adv_win_scale=1
net.core.netdev_max_backlog=30000
net.ipv4.tcp_max_syn_backlog=30000
net.ipv4.tcp_no_metrics_save=0
net.ipv4.tcp_congestion_control=htcp
net.ipv4.tcp_ecn=0
net.ipv4.tcp_fin_timeout=10
net.ipv4.tcp_keepalive_intvl=75
net.ipv4.tcp_keepalive_time=75
net.ipv4.tcp_tw_recycle=1
net.ipv4.tcp_tw_reuse=1
net.ipv4.ip_local_port_range=10000 64512
kernel.pid_max=262144
net.ipv6.route.max_size=65536
net.ipv4.tcp_mtu_probing=1
# Science DMZ NFS mods for ok storage
# vm.dirty_ratio = 2
# vm.dirty_background_ratio = 2
# Science DMZ NFS mods for good storage
vm.dirty_ratio = 0
vm.dirty_background_ratio = 0