cloudstore-developers
Changes
distributed-jmeter/conf/celeryd.conf 30(+30 -0)
distributed-jmeter/conf/confi.ini.production 11(+11 -0)
distributed-jmeter/conf/config.ini 11(+11 -0)
distributed-jmeter/conf/gunicorn.conf 15(+15 -0)
distributed-jmeter/conf/nginx.conf 108(+108 -0)
distributed-jmeter/conf/supervisor.conf 12(+12 -0)
distributed-jmeter/fabfile.py 150(+150 -0)
distributed-jmeter/requirements.txt 14(+14 -0)
Details
distributed-jmeter/conf/celeryd.conf 30(+30 -0)
diff --git a/distributed-jmeter/conf/celeryd.conf b/distributed-jmeter/conf/celeryd.conf
new file mode 100644
index 0000000..d3e8d84
--- /dev/null
+++ b/distributed-jmeter/conf/celeryd.conf
@@ -0,0 +1,30 @@
+; ==================================
+; celery worker supervisor example
+; ==================================
+
+[program:celery]
+; Set full path to celery program if using virtualenv
+command=/home/distributedjmeter/webapp/env/bin/celery worker -A webapp --loglevel=INFO
+
+directory=/home/distributedjmeter/webapp/releases/current/webapp
+user=distributedjmeter
+group=distributedjmeter
+numprocs=1
+stdout_logfile=/var/log/celery/distributedjmeter.log
+stderr_logfile=/var/log/celery/distributedjmeter.log
+autostart=true
+autorestart=true
+startsecs=10
+
+; Need to wait for currently executing tasks to finish at shutdown.
+; Increase this if you have very long running tasks.
+stopwaitsecs = 600
+
+; When resorting to send SIGKILL to the program to terminate it
+; send SIGKILL to its whole process group instead,
+; taking care of its children as well.
+killasgroup=true
+
+; if rabbitmq is supervised, set its priority higher
+; so it starts first
+priority=998
distributed-jmeter/conf/confi.ini.production 11(+11 -0)
diff --git a/distributed-jmeter/conf/confi.ini.production b/distributed-jmeter/conf/confi.ini.production
new file mode 100644
index 0000000..88cd19d
--- /dev/null
+++ b/distributed-jmeter/conf/confi.ini.production
@@ -0,0 +1,11 @@
+[EC2]
+aws_access_key_id =
+aws_secret_access_key =
+region = eu-west-1
+availability_zones = eu-west-1a
+ami_id = ami-480bea3f
+instance_type = t1.micro
+key_name = key-pair
+key_pair = /path/to/key-pair.pem
+num_jmeter_slaves = 2
+
distributed-jmeter/conf/config.ini 11(+11 -0)
diff --git a/distributed-jmeter/conf/config.ini b/distributed-jmeter/conf/config.ini
new file mode 100644
index 0000000..88cd19d
--- /dev/null
+++ b/distributed-jmeter/conf/config.ini
@@ -0,0 +1,11 @@
+[EC2]
+aws_access_key_id =
+aws_secret_access_key =
+region = eu-west-1
+availability_zones = eu-west-1a
+ami_id = ami-480bea3f
+instance_type = t1.micro
+key_name = key-pair
+key_pair = /path/to/key-pair.pem
+num_jmeter_slaves = 2
+
distributed-jmeter/conf/gunicorn.conf 15(+15 -0)
diff --git a/distributed-jmeter/conf/gunicorn.conf b/distributed-jmeter/conf/gunicorn.conf
new file mode 100644
index 0000000..9f71acb
--- /dev/null
+++ b/distributed-jmeter/conf/gunicorn.conf
@@ -0,0 +1,15 @@
+bind = '127.0.0.1:8001'
+backlog = 2048
+
+workers = 2
+worker_class = 'egg:gunicorn#sync'
+#worker_class = 'egg:gunicorn#gevent'
+worker_connections = 1000
+timeout = 30
+keepalive = 2
+
+user='distributedjmeter'
+group='distributedjmeter'
+
+debug = False
+spew = False
distributed-jmeter/conf/nginx.conf 108(+108 -0)
diff --git a/distributed-jmeter/conf/nginx.conf b/distributed-jmeter/conf/nginx.conf
new file mode 100644
index 0000000..0ebe29d
--- /dev/null
+++ b/distributed-jmeter/conf/nginx.conf
@@ -0,0 +1,108 @@
+upstream class_app_server {
+ server 127.0.0.1:8001 fail_timeout=0;
+}
+
+#server {
+# server_name cloudconference;
+# rewrite ^(.*) http://www.cloudconference.eu$1 permanent;
+#}
+
+server {
+ listen 80;
+ client_max_body_size 4G;
+ server_name cloudscale.xlab.si
+
+ keepalive_timeout 5;
+
+ access_log /var/log/nginx/distributedjmeter.access.log;
+ error_log /var/log/nginx/distributedjmeter.error.log;
+
+ root /var/www;
+
+ location / {
+ index index.html;
+ gzip on;
+ gzip_min_length 1000;
+ gzip_proxied expired no-cache no-store private auth;
+ gzip_types text/plain text/xml text/css application/xhtml+xml application/xml application/rss+xml application/javascript application/x-javascript;
+ gzip_disable "MSIE [1-6]\.";
+ }
+
+ location ~ /showcase/ {
+ include fastcgi_params;
+ fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+ fastcgi_pass 127.0.0.1:9001;
+ fastcgi_index index.php;
+ }
+
+ location /wiki {
+ proxy_pass http://127.0.0.1:81;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ }
+
+ location /distributed-jmeter/static {
+ alias /home/distributedjmeter/webapp/releases/current/webapp/static;
+ }
+
+ location /distributed-jmeter {
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header REMOTE_HOST $remote_addr;
+ proxy_set_header X-FORWARDED-PROTOCOL $scheme;
+ proxy_set_header SCRIPT_NAME /distributed-jmeter;
+ proxy_redirect off;
+ proxy_pass http://127.0.0.1:8001;
+ }
+
+ location /showcase-0 {
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header REMOTE_HOST $remote_addr;
+ proxy_set_header X-FORWARDED-PROTOCOL $scheme;
+ proxy_redirect off;
+ proxy_pass http://openstack.cloudscale.xlab.si/showcase-0;
+ }
+ location /showcase-1-a {
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header REMOTE_HOST $remote_addr;
+ proxy_set_header X-FORWARDED-PROTOCOL $scheme;
+ proxy_redirect off;
+ proxy_pass http://openstack.cloudscale.xlab.si/showcase-1-a;
+ }
+ location /showcase-1-b {
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header REMOTE_HOST $remote_addr;
+ proxy_set_header X-FORWARDED-PROTOCOL $scheme;
+ proxy_redirect off;
+ proxy_pass http://openstack.cloudscale.xlab.si/showcase-1-b;
+ }
+ location /jenkins {
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header REMOTE_HOST $remote_addr;
+ proxy_set_header X-FORWARDED-PROTOCOL $scheme;
+ proxy_redirect off;
+ proxy_pass http://localhost:8080;
+
+ }
+ location /sonar{
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header REMOTE_HOST $remote_addr;
+ proxy_set_header X-FORWARDED-PROTOCOL $scheme;
+ proxy_redirect off;
+ proxy_pass http://localhost:9000;
+
+ }
+
+}
distributed-jmeter/conf/supervisor.conf 12(+12 -0)
diff --git a/distributed-jmeter/conf/supervisor.conf b/distributed-jmeter/conf/supervisor.conf
new file mode 100644
index 0000000..a024879
--- /dev/null
+++ b/distributed-jmeter/conf/supervisor.conf
@@ -0,0 +1,12 @@
+[program:distributed_jmeter]
+user=distributedjmeter
+group=distributedjmeter
+directory=/home/distributedjmeter/webapp/releases/current/webapp
+command=/home/distributedjmeter/webapp/env/bin/python /home/distributedjmeter/webapp/releases/current/webapp/manage.py run_gunicorn -c /home/distributedjmeter/webapp/releases/current/webapp/conf/gunicorn.conf --error-logfile /var/log/gunicorn/distributedjmeter.log
+stderr_logfile=/var/log/supervisor/distributedjmeter.err.log
+stdout_logfile=/var/log/supervisor/distributedjmeter.log
+autostart=true
+autorestart=true
+redirect_stderr=True
+environment=HOME='/home/distributedjmeter/webapp/releases/current/webapp'
+
distributed-jmeter/fabfile.py 150(+150 -0)
diff --git a/distributed-jmeter/fabfile.py b/distributed-jmeter/fabfile.py
new file mode 100644
index 0000000..512e6ac
--- /dev/null
+++ b/distributed-jmeter/fabfile.py
@@ -0,0 +1,150 @@
+'''
+Created on 24. jul. 2012
+
+@author: urost
+'''
+from __future__ import with_statement
+from fabric.api import sudo, cd, run, settings, require, env, put, local, prefix, task
+from fabric.contrib.files import exists
+
+env.hosts = ['10.10.40.23'] #['pzs:30022']
+env.user = 'distributedjmeter'
+env.django_app = 'webapp'
+# tasks
+@task
+def new():
+ env.process_name = 'distributed_jmeter'
+ env.celery_process_name = 'celery'
+ env.user = 'distributedjmeter'
+ env.project_name = 'webapp'
+ env.postfix=''
+ env.path = '/home/%(user)s/%(project_name)s' % env
+ env.virtualhost_path = '%s%s' % (env.path, '/env')
+
+@task
+def deploy(install=False, migrate_db=False, branch=None):
+ """
+ Deploy the latest version (from env.project_branch or specified branch) of the site
+ to the servers, install any required third party modules,
+ install the virtual host (if forced with 'install=True', collect the static files,
+ symlink the media files and then restart the webserver.
+ """
+ import time
+ with settings(release = time.strftime('%Y%m%d%H%M%S')):
+ upload_tar_from_git(branch)
+ install_requirements()
+ symlink_current_release()
+ sudo('touch %(path)s/releases/current/scraper_app.log' % env)
+ sudo('chmod 777 %(path)s/releases/current/scraper_app.log' % env)
+
+ with cd('%(path)s/releases/current/%(project_name)s/conf/' % env):
+ run('cp confi.ini.production config.ini')
+
+ if install:
+ install_site()
+ migrate(install)
+ if migrate_db:
+ migrate(True)
+ #collect_static()
+ graceful_restart()
+ if install:
+ restart_webserver()
+
+@task
+def rollback():
+ """
+ Limited rollback capability. Simple loads the previously current
+ version of the code. Rolling back again will swap between the two.
+ """
+ require('path')
+ with cd('%(path)s/releases' % env):
+ run('mv current _previous;')
+ run('mv previous current;')
+ run('mv _previous previous;')
+ collect_static()
+ #symlink_media()
+ restart_webserver()
+
+
+# Helpers. These are called by other functions rather than directly
+def upload_tar_from_git(branch=None):
+ "Create an archive from the specified Git branch and upload it"
+ require('release')
+ require('path')
+ require('project_name')
+
+ with settings(archive_name = '%(release)s.tar.gz' % env):
+ local('/usr/bin/tar czf ../%(archive_name)s .' % env)
+
+ run('mkdir -p %(path)s/releases/%(release)s/%(project_name)s' % env)
+ put(('../%(archive_name)s' % env), ('%(path)s/packages/' % env))
+ with cd('%(path)s/releases/%(release)s/%(project_name)s' % env):
+ run('tar zxf ../../../packages/%(archive_name)s' % env)
+ with cd('%(path)s/releases/%(release)s/%(project_name)s/%(django_app)s' % env):
+ run('cp local_settings.py.production%(postfix)s local_settings.py' % env)
+ local('/bin/rm ../%(archive_name)s' % env)
+
+def install_site():
+ "Add the virtualhost file to nginx"
+ require('release')
+ require('project_name')
+ with cd('%(path)s/releases/%(release)s/%(project_name)s' % env):
+ sudo('cp conf/nginx.conf%(postfix)s /etc/nginx/sites-available/%(project_name)s' % env)
+ #sudo('cp conf/nginx.static.conf /etc/nginx/sites-available/%(project_name)s.static' % env)
+ if not exists('/etc/nginx/sites-enabled/%(project_name)s' % env):
+ with cd('/etc/nginx/sites-enabled'):
+ sudo('ln -s ../sites-available/%(project_name)s ./' % env)
+ #sudo('ln -s ../sites-available/%(user)s.static ./' % env)
+ with cd('%(path)s/releases/%(release)s/%(project_name)s/conf' % env):
+ sudo('cp supervisor.conf%(postfix)s /etc/supervisor/conf.d/%(project_name)s.conf' % env)
+ sudo('cp celeryd.conf%(postfix)s /etc/supervisor/conf.d/%(project_name)s.celeryd.conf' % env)
+
+
+def install_requirements():
+ "Install the required packages from the requirements file using pip"
+ require('release')
+ require('project_name')
+ with cd('%(virtualhost_path)s' % env):
+ with prefix('source %(virtualhost_path)s/bin/activate' % env):
+ run('%(virtualhost_path)s/bin/pip install -r %(path)s/releases/%(release)s/%(project_name)s/requirements.txt' % env)
+
+def symlink_current_release():
+ "Symlink our current release"
+ require('release')
+ with cd('%(path)s/releases' % env):
+ if exists('previous'):
+ run('rm previous')
+ if exists('current'):
+ run('mv current previous')
+ run('ln -s %(release)s current' % env)
+
+def collect_static():
+ "Collect static files in its folder"
+ require('project_name')
+ with cd('%(path)s/releases/current/%(project_name)s' % env):
+ with prefix('source %(virtualhost_path)s/bin/activate' % env):
+ run('%(virtualhost_path)s/bin/python manage.py collectstatic' % env)
+
+@task
+def migrate(install=False):
+ "Update the database"
+ require('project_name')
+ with cd('%(path)s/releases/current/%(project_name)s' % env):
+ with prefix('source %(virtualhost_path)s/bin/activate' % env):
+ if install:
+ run('%(virtualhost_path)s/bin/python manage.py syncdb --all' % env)
+ run('%(virtualhost_path)s/bin/python manage.py migrate --fake' % env)
+ else:
+ run('%(virtualhost_path)s/bin/python manage.py syncdb --all' % env)
+ run('%(virtualhost_path)s/bin/python manage.py migrate' % env)
+
+@task
+def restart_webserver():
+ "Restart the web server"
+ sudo('/etc/init.d/nginx reload')
+
+@task
+def graceful_restart():
+ "Restart the gunicorn processes"
+ sudo('supervisorctl restart %(process_name)s' % env)
+ sudo('supervisorctl restart %(celery_process_name)s' % env)
distributed-jmeter/requirements.txt 14(+14 -0)
diff --git a/distributed-jmeter/requirements.txt b/distributed-jmeter/requirements.txt
new file mode 100644
index 0000000..14cd8c3
--- /dev/null
+++ b/distributed-jmeter/requirements.txt
@@ -0,0 +1,14 @@
+Django==1.6.2
+MySQL-python==1.2.5
+amqp==1.4.3
+anyjson==0.3.3
+billiard==3.3.0.16
+boto==2.25.0
+celery==3.1.9
+ecdsa==0.10
+gunicorn==18.0
+kombu==3.0.12
+paramiko==1.12.1
+pycrypto==2.6.1
+pytz==2013.9
+wsgiref==0.1.2
diff --git a/distributed-jmeter/webapp/local_settings.py.production b/distributed-jmeter/webapp/local_settings.py.production
new file mode 100644
index 0000000..f2b2fbc
--- /dev/null
+++ b/distributed-jmeter/webapp/local_settings.py.production
@@ -0,0 +1,32 @@
+import os
+from settings import BASE_DIR
+
+DATABASES = {
+ 'default': {
+ 'ENGINE': 'django.db.backends.mysql',
+ 'NAME': 'load_test',
+ 'USER': 'user',
+ 'PASSWORD': 'password',
+ 'HOST': 'localhost',
+ 'PORT': '3306',
+ }
+}
+
+INSTALLED_APPS = (
+ 'django.contrib.admin',
+ 'django.contrib.auth',
+ 'django.contrib.contenttypes',
+ 'django.contrib.sessions',
+ 'django.contrib.messages',
+ 'django.contrib.staticfiles',
+ 'cloudscale',
+ 'gunicorn',
+)
+FORCE_SCRIPT_NAME='/distributed-jmeter/'
+URL_PREFIX = '/distributed-jmeter'
+MEDIA_ROOT = '{0}/../media/'.format(BASE_DIR)
+STATIC_ROOT = '{0}/../static/'.format(BASE_DIR)
+STATIC_URL = '{0}/static/'.format(URL_PREFIX)
+
+
+