Quartz for scheduling jobs with Java

The DAS Registry used to use a feature of the resin container which allowed servlets to be run at certain times of day/week/year or just run every few hours or days. This was achieved by setting run-at in the web.xml file of the webapp for the servlet.

As we are now moving to VMs and we are moving to tomcat for our production servers rather than resin this fascility does not exist. So after breifly considering rolling my own timer classes using the standard java classes I decided to check out the open source project Quartz (presumably named after the quartz watches timer mechanism).

I found the tutorials easy to get to grips with and within an hour or so had my first servlet converted to the new way of doing things.

We have 5 automated processes in the registry which are run at varying frequencies.

The things I needed to do were:

1) set up a quartz.properties file- and put in classpath

2) set up java class for Scheduling jobs in my case a servlet (so it could be initialized on tomcat startup using the servlet constructor).

3) create classes implementing the Job interface i.e. just having and execute method which will be called at the scheduled time.

4) there was already a database table to keep a track of when the last job of each type was run so there is persistence and I didnt’ need to use any of the quartz setup for that.

1) quartz.properties (put in resources dir which is on classpath set by eclipse build path):

org.quartz.scheduler.instanceName = RegistrySchedulerServlet
org.quartz.scheduler.instanceId = 1
org.quartz.scheduler.rmi.export = false
org.quartz.scheduler.rmi.proxy = false

org.quartz.threadPool.class = org.quartz.simpl.SimpleThreadPool
org.quartz.threadPool.threadCount = 5

org.quartz.jobStore.class = org.quartz.simpl.RAMJobStore

2) scheduler

package org.biojava.services.das.servlets;

import javax.servlet.http.HttpServlet;

import org.apache.log4j.Logger;
import org.quartz.JobDetail;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.quartz.Trigger;
import org.quartz.TriggerUtils;
import org.quartz.impl.StdSchedulerFactory;

import uk.ac.sanger.dasregistry.timer.Archiver;
import uk.ac.sanger.dasregistry.timer.AutoValidation;
import uk.ac.sanger.dasregistry.timer.CleanLogs;
import uk.ac.sanger.dasregistry.timer.LuceneIndexer;
import uk.ac.sanger.dasregistry.timer.Mirroring;

/**
* Servlet implementation class RegistrySchedulerServlet
*/
public class RegistrySchedulerServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
Scheduler scheduler;
public static Logger log=Logger.getLogger(” org.biojava.services.das.servlets”);

/**
* @see HttpServlet#HttpServlet()
*/
public RegistrySchedulerServlet() {
super();
try {
scheduler = StdSchedulerFactory.getDefaultScheduler();// default
// scheduler
// has been
// specified
// as this
// class in
// quartz.properties

// and start it off
scheduler.start();

// Define job instance
JobDetail indexerJob = new JobDetail(“indexing”, “group1”,
LuceneIndexer.class);
log.info(“indexing scheduled”);
JobDetail validationJob = new JobDetail(“autovalidation”, “group1”,
AutoValidation.class);
log.info(“autovalidation scheduled”);
JobDetail mirroringJob = new JobDetail(“mirroring”, “group1”,
Mirroring.class);
log.info(“mirrroring scheduled”);
JobDetail archiving = new JobDetail(“archiving”, “group1”,
Archiver.class);
log.info(“archiving scheduled”);
JobDetail cleanLogsJob = new JobDetail(“cleanLogs”, “group1”,
CleanLogs.class);
log.info(“clean logs scheduled”);
log.info(“scheduler setup”);
Trigger indexTrigger = TriggerUtils.makeDailyTrigger(“indexer”,
1, 30);//1.30am each day
// will go every 10 hours if set as only one machine in our cluster
// is allowed to run indexing
// comment out this servlet from develoment web.xml documents to
// prevent sending emails to some people
Trigger validationTrigger = TriggerUtils.makeHourlyTrigger(
“autovalidation”, 3, 1000000000);
// run every three hours or so – multiple machines can do this but
// only if last one longer than 3 hours ago (checked by DAO Admin
// checks that looks in database)
Trigger mirroringTrigger = TriggerUtils.makeWeeklyTrigger(
“mirroring”, 1, 1, 1);// run on Sunday at 1;01am
// jobs will be staggered as lastValidation etc times will be
// different.
Trigger archivingTrigger = TriggerUtils.makeDailyTrigger(
“archiving”, 4, 1); // 4:01am
// Trigger archivingTrigger =
// TriggerUtils.makeMinutelyTrigger(“archiving”, 4, 10); //4:01am
//Trigger cleanLogsTrigger=TriggerUtils.makeMinutelyTrigger(“cleanLogs”, 1,1000000);
Trigger cleanLogsTrigger=TriggerUtils.makeDailyTrigger(“cleanLogs”, 0,0);
// Schedule the job with the trigger
scheduler.scheduleJob(indexerJob, indexTrigger);
scheduler.scheduleJob(validationJob, validationTrigger);
scheduler.scheduleJob(mirroringJob, mirroringTrigger);
scheduler.scheduleJob(archiving, archivingTrigger);
scheduler.scheduleJob(cleanLogsJob, cleanLogsTrigger);

} catch (SchedulerException se) {
se.printStackTrace();
}

}

@Override
public void destroy() {
// TODO Auto-generated method stub
super.destroy();
try {
scheduler.shutdown();
System.out.println(“shutting down scheduler”);
} catch (SchedulerException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}

}

}

3) example class for indexing:

package uk.ac.sanger.dasregistry.timer;

import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import java.util.Timer;

import javax.servlet.ServletException;
import javax.sql.DataSource;

import org.apache.log4j.Logger;
import org.biojava.dasobert.dasregistry.DasSource;
import org.biojava.services.das.dao.DasSourceManager;
import org.biojava.services.das.registry.ConfigBean;
import org.biojava.services.das.registry.DasRegistrySql;
import org.biojava.services.das.servlets.ServletResponseIndexWriter;
import org.biojava.services.das2.Das1SourceDbProvider;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

public class LuceneIndexer implements Job {
private static final long serialVersionUID = 1L;

public static Logger logger = Logger
.getLogger(“uk.ac.sanger.dasregistry.timer.LuceneIndexer”);

int contactParentFrequency;
Timer timer;
DasSourceManager dao;

boolean showValidationLog = true;
public static final int ONEHOUR = 1000 * 60 * 60;

// public static final int ONEHOUR = 1000 * 60 ;

/**
* @params validationFrequency in milli-seconds
*/

public void execute(JobExecutionContext arg0) throws JobExecutionException {
dao = new DasSourceManager();
// String computerToRunIndexOn =
// getInitParameter(“computerToRunIndexOn”);
// System.out.println(“computer to run index on=” +
// computerToRunIndexOn);
String computerName = “”;
try {
computerName = InetAddress.getLocalHost().getHostName();
} catch (UnknownHostException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}

ApplicationContext context = new ClassPathXmlApplicationContext(
“config_dasregistry.xml”);

DasRegistrySql registry = (DasRegistrySql) context
.getBean(“registryBean”);

ConfigBean config = (ConfigBean) context.getBean(“configBean”);
showValidationLog = config.isPrintValidationStatus();
System.out.println(“computer name=” + computerName);
if (computerName.equals(“mib21789i”)) {
System.out.println(“running indexing on computer=” + computerName);
// put some code to test if a specific computer name – then index if
// not don’t do any of the tests
// if the correct machine then the files can then be picked up by
// the getDasregistryIndexes.sh script that is run by a cron

Connection conn = null;
boolean doIndexing = false;
try {

DataSource dataSource = registry.getDataSource();
conn = dataSource.getConnection();

doIndexing = dao.doAdminChecks(conn, “lastIndex”,
“indexingStarted”, “indexing”);

} catch (Exception e) {
e.printStackTrace();
} finally {
if (conn != null) {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}

if (!doIndexing) {
logger
.info(” not running indexing due to dao admin checks… “);
return;
}

logger.info(“running auto indexing …”);
// String directory = getServletContext().getRealPath(“/”);

// System.out.println(“dir=” + directory);
try {
index(“/”);
} catch (ServletException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
releaseIndexingLock(registry);

registry = null;
}

}

private void releaseIndexingLock(DasRegistrySql registry) {
System.out.println(“releasing indexing lock”);
DataSource dataSource = registry.getDataSource();

Connection conn = null;
// boolean doValidate = false;
try {
conn = dataSource.getConnection();
dao.releaseAdminLock(conn, “indexing”, “lastIndex”);
// as we now update the valid_capabilities tables we want
// to tell the sources.xml returned by the registry it will need
// updating etc
// dao.pingLastModified(conn);

} catch (Exception e) {
e.printStackTrace();
} finally {
if (conn != null) {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
}

private void index(String directory) throws ServletException, IOException {
System.out.println(“index called in Das2RegistryServlet”);
Das1SourceDbProvider das1provider = new Das1SourceDbProvider();

List<DasSource> sources = das1provider.getDasSources();
Map<String, List<Map<String, String>>> types = das1provider
.getAutoIdWithTypes(sources);

ServletResponseIndexWriter writer = new ServletResponseIndexWriter(
directory);

writer.writeIndexResponse(sources, types);

}

}

Advertisements
  1. No trackbacks yet.

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s

%d bloggers like this: