Skip to end of metadata
Go to start of metadata

Set up:

  • Db server: MySQL 5.5.44 (from apt-get) m3.large single instance
  • 4 App nodes m3.medium
  • HAProxy as Load balancer (HAProxy version 1.4.24 from apt-get) m3.medium

All instances running Ubuntu Server 14.04

MySQL my.cfg

# 
# The MySQL database server configuration file. 
# 
# You can copy this to one of: 
# - "/etc/mysql/my.cnf" to set global options, 
# - "~/.my.cnf" to set user-specific options. 
# 
# One can use all long options that the program supports. 
# Run program with --help to get a list of available options and with 
# --print-defaults to see which it would actually understand and use. 
# 
# For explanations see 
# http://dev.mysql.com/doc/mysql/en/server-system-variables.html 
 
# This will be passed to all mysql clients 
# It has been reported that passwords should be enclosed with ticks/quotes 
# escpecially if they contain "#" chars... 
# Remember to edit /etc/mysql/debian.cnf when changing the socket location. 
[client] 
port            = 3306 
socket          = /var/run/mysqld/mysqld.sock 
 
# Here is entries for some specific programs 
# The following values assume you have at least 32M ram 
 
# This was formally known as [safe_mysqld]. Both versions are currently parsed. 
[mysqld_safe] 
socket          = /var/run/mysqld/mysqld.sock 
nice            = 0 
 
 
[mysqld] 
transaction-isolation=READ-COMMITTED 
default_storage_engine = InnoDB 
sql-mode="STRICT_TRANS_TABLES,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION" 
# 
# * Basic Settings 
# 
user            = mysql 
pid-file        = /var/run/mysqld/mysqld.pid 
socket          = /var/run/mysqld/mysqld.sock 
port            = 3306 
basedir         = /usr 
datadir         = /var/lib/mysql 
tmpdir          = /tmp 
lc-messages-dir = /usr/share/mysql 
skip-external-locking 
# 
# Instead of skip-networking the default is now to listen only on 
# localhost which is more compatible and is not less secure. 
bind-address            = 10.33.168.41 
# 
# * Fine Tuning 
# 
key_buffer              = 10M 
max_allowed_packet      = 32M 
thread_stack            = 192K 
thread_cache_size       = 64 
# This replaces the startup script and checks MyISAM tables if needed 
# the first time they are touched 
myisam-recover         = BACKUP 
max_connections        = 850 
#table_cache            = 64 
#thread_concurrency     = 10 
# 
# * Query Cache Configuration 
# 
query_cache_limit       = 16M 
query_cache_size        = 64M 
 
 
#Innodb 
 
innodb_buffer_pool_size = 3G 
innodb_log_file_size    = 256M 
innodb_log_buffer_size  = 4M 
innodb_flush_log_at_trx_commit  = 2 
innodb_thread_concurrency       = 24 
innodb_flush_method     = O_DIRECT 
innodb_file_per_table   = 1 
 
 
# 
# * Logging and Replication 
# 
# Both location gets rotated by the cronjob. 
# Be aware that this log type is a performance killer. 
# As of 5.1 you can enable the log at runtime! 
#general_log_file        = /var/log/mysql/mysql.log 
#general_log             = 1 
# 
# Error log - should be very few entries. 
# 
log_error = /var/log/mysql/error.log 
# 
# Here you can see queries with especially long duration 
log_slow_queries        = /var/log/mysql/mysql-slow.log 
long_query_time = 2 
log-queries-not-using-indexes = 0 
# 
# The following can be used as easy to replay backup logs or for replication. 
# note: if you are setting up a replication slave, see README.Debian about 
#       other settings you may need to change. 
#server-id              = 1 
#log_bin                        = /var/log/mysql/mysql-bin.log 
expire_logs_days        = 10 
max_binlog_size         = 100M 
#binlog_do_db           = include_database_name 
#binlog_ignore_db       = include_database_name 
# 
# * InnoDB 
# 
# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 
# Read the manual for more InnoDB related options. There are many! 
# 
# * Security Features 
# 
# Read the manual, too, if you want chroot! 
# chroot = /var/lib/mysql/ 
# 
# For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 
# 
# ssl-ca=/etc/mysql/cacert.pem 
# ssl-cert=/etc/mysql/server-cert.pem 
# ssl-key=/etc/mysql/server-key.pem 
 
[mysqldump] 
quick 
quote-names 
max_allowed_packet      = 16M 
hex_blob                = 1 
 
[mysql] 
#no-auto-rehash # faster start of mysql but no tab completition 
 
[isamchk] 
key_buffer              = 16M 
 
# 
# * IMPORTANT: Additional settings that can override those from this file! 
#   The files must end with '.cnf', otherwise they'll be ignored. 
# 
!includedir /etc/mysql/conf.d/ 

Relevant jboss configuration

server.xml

...
   <Service name="jboss.web">

      <!-- A HTTP/1.1 Connector on port 8080 -->

      <Connector protocol="HTTP/1.1" port="8080" address="${jboss.bind.address}" URIEncoding="UTF-8"
        maxThreads="400" maxHttpHeaderSize="8192"  acceptCount="400"
        connectionTimeout="20000"  connectionUploadTimeout="300000" disableUploadTimeout="true"
        minSpareThreads="25" maxSpareThreads="75"
        emptySessionPath="true" enableLookups="false" redirectPort="8443" />
...

mysql-ds.xml

<?xml version="1.0" encoding="UTF-8"?>

<!-- $Id: mysql-ds.xml,v 1.2 2005/03/10 02:26:34 cperfect Exp $ -->
<!--  Datasource config for MySQL using 3.0.9 available from:
http://www.mysql.com/downloads/api-jdbc-stable.html
-->

<datasources>
  <!-- lams-ds datasource for use by LAMS Core and Services -->
  <local-tx-datasource>
    <jndi-name>jdbc/lams-ds</jndi-name>
    <connection-url>jdbc:mysql://10.33.168.41:3306/lams</connection-url>
    <driver-class>com.mysql.jdbc.Driver</driver-class>
    <!-- Tell MySQL Connector to use UTF-8 character encoding, -->
    <!-- and convert MySQL zero times to null -->
    <connection-property name="characterEncoding">utf8</connection-property>
    <connection-property name="zeroDateTimeBehavior">convertToNull</connection-property>
    <!-- Set transaction isolation level to match db -->
    <transaction-isolation>TRANSACTION_READ_COMMITTED</transaction-isolation>
    <user-name>lams</user-name>
    <password>lamsdemo</password>
    <!-- Set minimum adn maximum size of pool -->
    <max-pool-size>250</max-pool-size>
    <min-pool-size>0</min-pool-size>
    <!-- Connections not used after 5 minutes will be closed and removed -->
    <idle-timeout-minutes>5</idle-timeout-minutes>
  </local-tx-datasource>
</datasources>

HAProxy

I love HAProxy, I cannot recommend it enough. An awesome piece of software!

# this config needs haproxy-1.1.28 or haproxy-1.2.1

global
        log 127.0.0.1   local0
        log 127.0.0.1   local1 notice
        #log loghost    local0 info
        maxconn 4096
        #chroot /usr/share/haproxy
        user haproxy
        group haproxy
        daemon
        #debug
        #quiet

defaults
        log     global
        mode    http
        option  httplog
        option  dontlognull
        retries 3
        option redispatch
        maxconn 2000
        contimeout      5000
        clitimeout      50000
        srvtimeout      50000
       stats enable
       stats auth someuser:somepassword
       stats realm Haproxy\ Statistics
       stats uri /haproxy?stats

listen http_proxy 10.7.181.228:80
        default_backend servers

backend servers
       mode http
       balance roundrobin
       cookie JSESSIONID prefix
       option httpclose
       appsession JSESSIONID len 64 timeout 3h request-learn
       option http-server-close
       option forwardfor
       #option httpchk HEAD /lams/www/news.html HTTP/1.0
       server web1 10.140.27.196:8080 cookie haci_web1 check maxconn 450
       server web2 10.158.150.21:8080 cookie haci_web2 check maxconn 450
       server web3 10.158.147.45:8080 cookie haci_web3 check maxconn 450
       server web4 10.168.119.75:8080 cookie haci_web4 check maxconn 450

Test harness set up

Pretty much standard set up and using a TBL lesson (very intensive load!)

To ensure that we mimic some what realistic load, I run the Th with this script:

$ r=$(( $RANDOM % 300 )); echo $r; sleep $r; ant run

This will automatically start the test harness at a random time within 0 and 300 seconds (5 mins).

Basic configuration

learnerTest1.properties

...
NumberOfLearners = 110
...
MinDelay = 5
MaxDelay = 20
...

Results

TH instance

Average response rate (seconds)

01

0.6062727272727273

02

0.823690909090909

03

0.8826545454545455

04

0.6258264115725618

05

0.7083841751548882

06

0.6396909493384314

07

0.5536065857885615

08

0.6676090909090909

09

0.6377040669022848

10

0.5363063278469752

Pretty good results for just a small cluster

  • No labels