From 47bb1e3845ba951db5373e4c9f9e164793bfd067 Mon Sep 17 00:00:00 2001 From: Matzz Date: Mon, 14 Jul 2014 14:31:31 +0200 Subject: [PATCH 01/29] Delayed queue, not tested yet --- .../queues/AbstractBlockingQueue.java | 12 ++ .../queues/MySQLBasedDelayQueue.java | 125 ++++++++++++++ .../db_patterns/queues/MySQLBasedQueue.java | 158 ++++++++++++------ .../serializator/DefaultSerializator.java | 58 +++++++ .../net/bramp/serializator/Serializator.java | 6 + .../queues/MySQLBasedDelayQueueTests.java | 138 +++++++++++++++ 6 files changed, 449 insertions(+), 48 deletions(-) create mode 100644 src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java create mode 100644 src/main/java/net/bramp/serializator/DefaultSerializator.java create mode 100644 src/main/java/net/bramp/serializator/Serializator.java create mode 100644 src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java diff --git a/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java b/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java index 60c942e..40d97f2 100644 --- a/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java @@ -137,4 +137,16 @@ public boolean removeAll(Collection c) { public boolean retainAll(Collection c) { throw new UnsupportedOperationException(); } + + protected abstract String getAddQuery(); + + protected abstract String getPeekQuery(); + + protected abstract String[] getPollQuery(); + + protected abstract String getSizeQuery(); + + protected abstract String getCleanupQuery(); + + protected abstract String getCleanupAllQuery(); } diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java new file mode 100644 index 0000000..6c543b0 --- /dev/null +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java @@ -0,0 +1,125 @@ +package net.bramp.db_patterns.queues; + +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Date; +import java.util.concurrent.Delayed; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; + +import javax.sql.DataSource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.bramp.db_patterns.locks.MySQLSleepBasedCondition; +import net.bramp.serializator.Serializator; + +/** + * A queue backed by MySQL + *

+ * CREATE TABLE queue ( + * id INT UNSIGNED NOT NULL AUTO_INCREMENT, + * queue_name VARCHAR(255) NOT NULL, -- Queue name + * inserted TIMESTAMP NOT NULL, -- Time the row was inserted + * inserted_by VARCHAR(255) NOT NULL, -- and by who + * acquired TIMESTAMP NULL, -- Time the row was acquired + * acquired_by VARCHAR(255) NULL, -- and by who + * delayed_to TIMESTAMP NULL, + * value BLOB NOT NULL, -- The actual data + * PRIMARY KEY (id) + * ) ENGINE=INNODB DEFAULT CHARSET=UTF8; + *

+ * TODO Create efficient drainTo + * + * @param + * @author bramp + */ +public class MySQLBasedDelayQueue extends MySQLBasedQueue { + + + final static protected String delayedAddQuery = "INSERT INTO queue (queue_name, inserted, inserted_by, delayed_to, value) values (?, now(), ?, DATE_ADD(NOW(), INTERVAL ? SECOND), ?)"; + final static protected String delayedPeekQuery = "SELECT value FROM queue WHERE acquired IS NULL AND (delayed_to<=now() OR delayed_to is null) AND queue_name = ? ORDER BY id ASC LIMIT 1"; + final static String delayedPollQuery[] = { + "SET @update_id := -1; ", + "UPDATE queue SET " + + " id = (SELECT @update_id := id), " + + " acquired = NOW(), " + + " acquired_by = ? " + + "WHERE "+ + "acquired IS NULL AND " + + "(delayed_to<=now() OR delayed_to is null) AND "+ + "queue_name = ? " + + "ORDER BY id ASC " + + "LIMIT 1; ", + "SELECT value FROM queue WHERE id = @update_id" + }; + + public MySQLBasedDelayQueue(DataSource ds, String queueName, Class type, String me) { + super(ds, queueName, type, me); + } + + public MySQLBasedDelayQueue(DataSource ds, String queueName, Serializator serializator, String me) { + super(ds, queueName, serializator, me); + } + + public boolean add(E value) { + try { + Connection c = ds.getConnection(); + try { + PreparedStatement s = c.prepareStatement(getAddQuery()); + try { + + + s.setString(1, queueName); + s.setObject(2, me); // Inserted by me + s.setLong(3, value.getDelay(TimeUnit.SECONDS)); + setValueToStatment(s, 4, value); + s.execute(); + + // Wake up one + condition.signal(); + + return true; + + } finally { + s.close(); + } + } finally { + c.close(); + } + + } catch (SQLException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + } + + @Override + protected String getAddQuery() { + return delayedAddQuery; + } + + @Override + protected String getPeekQuery() { + return delayedPeekQuery; + } + + @Override + protected String[] getPollQuery() { + return pollQuery; + } + + @Override + protected String getCleanupQuery() { + return cleanupQuery; + } + + @Override + protected String getCleanupAllQuery() { + return cleanupAllQuery; + } +} diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java index b9b417f..3ea3eca 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java @@ -1,5 +1,6 @@ package net.bramp.db_patterns.queues; +import java.io.IOException; import java.sql.CallableStatement; import java.sql.Connection; import java.sql.PreparedStatement; @@ -15,23 +16,20 @@ import org.slf4j.LoggerFactory; import net.bramp.db_patterns.locks.MySQLSleepBasedCondition; +import net.bramp.serializator.Serializator; /** * A queue backed by MySQL *

- * CREATE TABLE queue ( - * id INT UNSIGNED NOT NULL AUTO_INCREMENT, - * queue_name VARCHAR(255) NOT NULL, -- Queue name - * inserted TIMESTAMP NOT NULL, -- Time the row was inserted - * inserted_by VARCHAR(255) NOT NULL, -- and by who - * acquired TIMESTAMP NULL, -- Time the row was acquired - * acquired_by VARCHAR(255) NULL, -- and by who - * value BLOB NOT NULL, -- The actual data - * PRIMARY KEY (id) - * ) ENGINE=INNODB DEFAULT CHARSET=UTF8; + * CREATE TABLE queue ( id INT UNSIGNED NOT NULL AUTO_INCREMENT, queue_name + * VARCHAR(255) NOT NULL, -- Queue name inserted TIMESTAMP NOT NULL, -- Time the + * row was inserted inserted_by VARCHAR(255) NOT NULL, -- and by who acquired + * TIMESTAMP NULL, -- Time the row was acquired acquired_by VARCHAR(255) NULL, + * -- and by who value BLOB NOT NULL, -- The actual data PRIMARY KEY (id) ) + * ENGINE=INNODB DEFAULT CHARSET=UTF8; *

* TODO Create efficient drainTo - * + * * @param * @author bramp */ @@ -39,7 +37,7 @@ public class MySQLBasedQueue extends AbstractBlockingQueue { final static Logger LOG = LoggerFactory.getLogger(MySQLBasedQueue.class); - final static String addQuery = "INSERT INTO queue (queue_name, inserted, inserted_by, value) values (?, now(), ?, ?)"; + final static String addQuery = "INSERT INTO queue (queue_name, inserted, inserted_by, value) values (?, now(), ?, ?)"; final static String peekQuery = "SELECT value FROM queue WHERE acquired IS NULL AND queue_name = ? ORDER BY id ASC LIMIT 1"; final static String sizeQuery = "SELECT COUNT(*) FROM queue WHERE acquired IS NULL AND queue_name = ?"; @@ -49,48 +47,60 @@ public class MySQLBasedQueue extends AbstractBlockingQueue { final static String pollQuery[] = { "SET @update_id := -1; ", - "UPDATE queue SET " + - " id = (SELECT @update_id := id), " + - " acquired = NOW(), " + - " acquired_by = ? " + - "WHERE acquired IS NULL AND queue_name = ? " + - "ORDER BY id ASC " + - "LIMIT 1; ", + "UPDATE queue SET " + " id = (SELECT @update_id := id), " + + " acquired = NOW(), " + " acquired_by = ? " + + "WHERE acquired IS NULL AND queue_name = ? " + + "ORDER BY id ASC " + "LIMIT 1; ", - "SELECT value FROM queue WHERE id = @update_id" - }; + "SELECT value FROM queue WHERE id = @update_id" }; - final static String cleanupQuery = - "DELETE FROM queue " + - "WHERE acquired IS NOT NULL " + - " AND queue_name = ? " + - " AND acquired < DATE_SUB(NOW(), INTERVAL 10 DAY)"; + final static String cleanupQuery = "DELETE FROM queue " + + "WHERE acquired IS NOT NULL " + " AND queue_name = ? " + + " AND acquired < DATE_SUB(NOW(), INTERVAL 10 DAY)"; - final static String cleanupAllQuery = - "DELETE FROM queue " + - "WHERE acquired IS NOT NULL " + - " AND acquired < DATE_SUB(NOW(), INTERVAL 10 DAY)"; + final static String cleanupAllQuery = "DELETE FROM queue " + + "WHERE acquired IS NOT NULL " + + " AND acquired < DATE_SUB(NOW(), INTERVAL 10 DAY)"; final String me; final DataSource ds; final String queueName; - final Class type; + + private Class type = null; + private Serializator serializator = null; final Condition condition; /** * Creates a new MySQL backed queue - * + * * @param ds * @param queueName * @param type - * @param me The name of this node, for storing in the database table + * @param me The name of this node, for storing in the database table */ public MySQLBasedQueue(DataSource ds, String queueName, Class type, String me) { + this(ds, queueName, me); + this.type = type; + } + + /** + * Creates a new MySQL backed queue + * + * @param ds + * @param queueName + * @param serializator + * @param me The name of this node, for storing in the database table + */ + public MySQLBasedQueue(DataSource ds, String queueName, Serializator serializator, String me) { + this(ds, queueName, me); + this.serializator = serializator; + } + + protected MySQLBasedQueue(DataSource ds, String queueName, String me) { this.ds = ds; this.queueName = queueName; - this.type = type; this.condition = new MySQLSleepBasedCondition(ds, "queue-" + queueName); this.me = me; } @@ -99,11 +109,11 @@ public boolean add(E value) { try { Connection c = ds.getConnection(); try { - PreparedStatement s = c.prepareStatement(addQuery); + PreparedStatement s = c.prepareStatement(getAddQuery()); try { s.setString(1, queueName); s.setObject(2, me); // Inserted by me - s.setObject(3, value); + setValueToStatment(s, 3, value); s.execute(); // Wake up one @@ -130,13 +140,13 @@ public E peek() { try { Connection c = ds.getConnection(); try { - PreparedStatement s = c.prepareStatement(peekQuery); + PreparedStatement s = c.prepareStatement(getPeekQuery()); try { s.setString(1, queueName); if (s.execute()) { ResultSet rs = s.getResultSet(); if (rs != null && rs.next()) { - return rs.getObject(1, type); + return getValueFromResult(rs, 1); } } @@ -160,6 +170,7 @@ public E peek() { public E poll() { try { Connection c = ds.getConnection(); + String[] pollQuery = getPollQuery(); try { c.setAutoCommit(false); @@ -179,7 +190,7 @@ public E poll() { if (s3.execute()) { ResultSet rs = s3.getResultSet(); if (rs != null && rs.next()) { - return rs.getObject(1, type); + return getValueFromResult(rs, 1); } } @@ -199,7 +210,7 @@ public int size() { try { Connection c = ds.getConnection(); try { - PreparedStatement s = c.prepareStatement(sizeQuery); + PreparedStatement s = c.prepareStatement(getSizeQuery()); s.setString(1, queueName); s.execute(); @@ -219,12 +230,13 @@ public int size() { } /** - * Blocks until something is in the queue, up to timeout - * null if timeout occurs + * Blocks until something is in the queue, up to timeout null if timeout + * occurs */ public E poll(long timeout, TimeUnit unit) throws InterruptedException { - final long deadlineMillis = System.currentTimeMillis() + unit.toMillis(timeout); + final long deadlineMillis = System.currentTimeMillis() + + unit.toMillis(timeout); final Date deadline = new Date(deadlineMillis); E head = null; @@ -237,8 +249,10 @@ public E poll(long timeout, TimeUnit unit) throws InterruptedException { break; // Block until we are woken, or deadline - // Because we don't have a distributed lock around this condition, there is a race condition - // whereby we might miss a notify(). However, we can somewhat mitigate the problem, by using + // Because we don't have a distributed lock around this condition, + // there is a race condition + // whereby we might miss a notify(). However, we can somewhat + // mitigate the problem, by using // this in a polling fashion stillWaiting = condition.awaitUntil(deadline); } @@ -249,7 +263,7 @@ public E poll(long timeout, TimeUnit unit) throws InterruptedException { public void cleanup() throws SQLException { Connection c = ds.getConnection(); try { - CallableStatement s = c.prepareCall(cleanupQuery); + CallableStatement s = c.prepareCall(getCleanupAllQuery()); s.setString(1, queueName); s.execute(); @@ -260,17 +274,65 @@ public void cleanup() throws SQLException { /** * Cleans up all queues - * + * * @throws SQLException */ public void cleanupAll() throws SQLException { Connection c = ds.getConnection(); try { - CallableStatement s = c.prepareCall(cleanupAllQuery); + CallableStatement s = c.prepareCall(getCleanupAllQuery()); s.execute(); } finally { c.close(); } } + + @Override + protected String getAddQuery() { + return addQuery; + } + + @Override + protected String getPeekQuery() { + return peekQuery; + } + + @Override + protected String[] getPollQuery() { + return pollQuery; + } + + @Override + protected String getSizeQuery() { + return sizeQuery; + } + + @Override + protected String getCleanupQuery() { + return cleanupQuery; + } + + @Override + protected String getCleanupAllQuery() { + return cleanupAllQuery; + } + + protected E getValueFromResult(ResultSet rs, int index) throws SQLException { + if(serializator == null) { + return rs.getObject(1, type); + } + else { + return serializator.deserialize(rs.getBytes(index)); + } + } + + protected void setValueToStatment(PreparedStatement s, int index, E obj) throws SQLException { + if(serializator == null) { + s.setObject(index, obj); + } + else { + s.setBytes(index, serializator.serialize(obj)); + } + } } diff --git a/src/main/java/net/bramp/serializator/DefaultSerializator.java b/src/main/java/net/bramp/serializator/DefaultSerializator.java new file mode 100644 index 0000000..8d109f9 --- /dev/null +++ b/src/main/java/net/bramp/serializator/DefaultSerializator.java @@ -0,0 +1,58 @@ +package net.bramp.serializator; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + +public class DefaultSerializator implements Serializator { + + @Override + public byte[] serialize(E obj) { + byte[] array = null; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + try { + ObjectOutputStream objectOut = new ObjectOutputStream(out); + try { + objectOut.writeObject(obj); + array = out.toByteArray(); + } catch (IOException e) { + objectOut.close(); + } + } finally { + out.close(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + + return array; + } + + @Override + public E deserialize(byte[] bytes) { + E obj = null; + ByteArrayInputStream in = new ByteArrayInputStream(bytes); + try { + try { + ObjectInputStream objectIn = new ObjectInputStream(in); + try { + obj = (E) objectIn.readObject(); + } catch (IOException e) { + objectIn.close(); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } finally { + in.close(); + } + + } catch (IOException e) { + throw new RuntimeException(e); + } + return obj; + } + +} diff --git a/src/main/java/net/bramp/serializator/Serializator.java b/src/main/java/net/bramp/serializator/Serializator.java new file mode 100644 index 0000000..d328861 --- /dev/null +++ b/src/main/java/net/bramp/serializator/Serializator.java @@ -0,0 +1,6 @@ +package net.bramp.serializator; + +public interface Serializator { + public byte[] serialize(E obj); + public E deserialize(byte[] bytes); +} diff --git a/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java b/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java new file mode 100644 index 0000000..db228b0 --- /dev/null +++ b/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java @@ -0,0 +1,138 @@ +package net.bramp.db_patterns.queues; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.io.Serializable; +import java.sql.SQLException; +import java.util.concurrent.Delayed; +import java.util.concurrent.TimeUnit; + +import javax.sql.DataSource; + +import net.bramp.db_patterns.DatabaseUtils; +import net.bramp.serializator.DefaultSerializator; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class MySQLBasedDelayQueueTests { + + final static long WAIT_FOR_TIMING_TEST = 300; // in ms + + private String queueName; + private DataSource ds; + + private MySQLBasedQueue queue; + + @Before + public void setup() { + // Different queue name for each test (to avoid test clashes) + queueName = java.util.UUID.randomUUID().toString(); + ds = DatabaseUtils.createDataSource(); + + queue = new MySQLBasedDelayQueue(ds, queueName, new DefaultSerializator(), "test"); + } + + @After + public void cleanupDatabase() throws SQLException { + queue.clear(); + queue.cleanupAll(); + assertEmpty(); + } + + protected void assertEmpty() { + assertTrue("Queue should start empty", queue.isEmpty()); + assertEquals("Queue should start empty", 0, queue.size()); + assertNull("Queue head should be null", queue.peek()); + } + + @Test + public void test() throws IOException { + + assertEmpty(); + + DelayedString a = new DelayedString("A", 0); + DelayedString b = new DelayedString("B", 0); + + assertTrue( queue.add(new DelayedString("A", 0)) ); + + assertEquals("Queue should contain one item", 1, queue.size()); + assertEquals("Queue head should be A", a, queue.peek()); + + assertTrue( queue.add(b) ); + + assertEquals("Queue should start empty", 2, queue.size()); + assertEquals("Queue head should be A", a, queue.peek()); + + assertEquals("Queue head should be A", a, queue.poll()); + + assertEquals("Queue should start empty", 1, queue.size()); + assertEquals("Queue head should be B", b, queue.peek()); + + assertEquals("Queue head should be B", b, queue.poll()); + + assertEmpty(); + } + + @Test(timeout=1000) + public void pollBlockingTest() throws InterruptedException { + assertEmpty(); + + long wait = WAIT_FOR_TIMING_TEST; + + long now = System.currentTimeMillis(); + String ret = queue.poll(wait, TimeUnit.MILLISECONDS).get(); + long duration = System.currentTimeMillis() - now; + + assertNull("poll timed out", ret); + + assertTrue("We waited less than " + wait + "ms (actual:" + duration + ")", duration >= wait); + assertTrue("We waited more than " + (wait*1.2) + "ms (actual:" + duration + ")", duration < wait * 1.2); + + assertEmpty(); + } + + protected static class DelayedString implements Delayed, Serializable { + + private static final long serialVersionUID = -574306132564575817L; + + private String str; + private long time; + private transient TimeUnit unit = TimeUnit.NANOSECONDS; + + public DelayedString(String str, long seconds) { + this.str = str; + this.time = seconds + System.nanoTime(); + } + + @Override + public int compareTo(Delayed o) { + Long l = o.getDelay(unit); + return l.compareTo(this.time); + } + + public boolean equalsTo(Delayed o) { + if(o instanceof DelayedString) { + DelayedString v = (DelayedString) o; + return get().equals(v.get()); + } + else { + return false; + } + } + + + @Override + public long getDelay(TimeUnit unit) { + return unit.convert(time - System.nanoTime(), unit); + } + + public String get() { + return str; + } + } +} From 752e45cc07ce8641dee2fbb974d8a098f147aa72 Mon Sep 17 00:00:00 2001 From: Matzz Date: Mon, 14 Jul 2014 14:38:01 +0200 Subject: [PATCH 02/29] formatting fix --- .../db_patterns/queues/MySQLBasedQueue.java | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java index 3ea3eca..9a8e5c3 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java @@ -21,15 +21,19 @@ /** * A queue backed by MySQL *

- * CREATE TABLE queue ( id INT UNSIGNED NOT NULL AUTO_INCREMENT, queue_name - * VARCHAR(255) NOT NULL, -- Queue name inserted TIMESTAMP NOT NULL, -- Time the - * row was inserted inserted_by VARCHAR(255) NOT NULL, -- and by who acquired - * TIMESTAMP NULL, -- Time the row was acquired acquired_by VARCHAR(255) NULL, - * -- and by who value BLOB NOT NULL, -- The actual data PRIMARY KEY (id) ) - * ENGINE=INNODB DEFAULT CHARSET=UTF8; + * CREATE TABLE queue ( + * id INT UNSIGNED NOT NULL AUTO_INCREMENT, + * queue_name VARCHAR(255) NOT NULL, -- Queue name + * inserted TIMESTAMP NOT NULL, -- Time the row was inserted + * inserted_by VARCHAR(255) NOT NULL, -- and by who + * acquired TIMESTAMP NULL, -- Time the row was acquired + * acquired_by VARCHAR(255) NULL, -- and by who + * value BLOB NOT NULL, -- The actual data + * PRIMARY KEY (id) + * ) ENGINE=INNODB DEFAULT CHARSET=UTF8; *

* TODO Create efficient drainTo - * + * * @param * @author bramp */ @@ -67,8 +71,8 @@ public class MySQLBasedQueue extends AbstractBlockingQueue { final DataSource ds; final String queueName; - private Class type = null; - private Serializator serializator = null; + protected Class type = null; + protected Serializator serializator = null; final Condition condition; @@ -235,8 +239,7 @@ public int size() { */ public E poll(long timeout, TimeUnit unit) throws InterruptedException { - final long deadlineMillis = System.currentTimeMillis() - + unit.toMillis(timeout); + final long deadlineMillis = System.currentTimeMillis() + unit.toMillis(timeout); final Date deadline = new Date(deadlineMillis); E head = null; @@ -249,10 +252,8 @@ public E poll(long timeout, TimeUnit unit) throws InterruptedException { break; // Block until we are woken, or deadline - // Because we don't have a distributed lock around this condition, - // there is a race condition - // whereby we might miss a notify(). However, we can somewhat - // mitigate the problem, by using + // Because we don't have a distributed lock around this condition, there is a race condition + // whereby we might miss a notify(). However, we can somewhat mitigate the problem, by using // this in a polling fashion stillWaiting = condition.awaitUntil(deadline); } From 32aef99ee6e3dec87b658c5e0c8d21333b20516c Mon Sep 17 00:00:00 2001 From: Matzz Date: Mon, 14 Jul 2014 14:40:21 +0200 Subject: [PATCH 03/29] formatting fix2 --- .../java/net/bramp/db_patterns/queues/MySQLBasedQueue.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java index 9a8e5c3..249ec6c 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java @@ -56,7 +56,8 @@ public class MySQLBasedQueue extends AbstractBlockingQueue { + "WHERE acquired IS NULL AND queue_name = ? " + "ORDER BY id ASC " + "LIMIT 1; ", - "SELECT value FROM queue WHERE id = @update_id" }; + "SELECT value FROM queue WHERE id = @update_id" + }; final static String cleanupQuery = "DELETE FROM queue " + "WHERE acquired IS NOT NULL " + " AND queue_name = ? " @@ -82,7 +83,7 @@ public class MySQLBasedQueue extends AbstractBlockingQueue { * @param ds * @param queueName * @param type - * @param me The name of this node, for storing in the database table + * @param me The name of this node, for storing in the database table */ public MySQLBasedQueue(DataSource ds, String queueName, Class type, String me) { this(ds, queueName, me); From fb128c1c5898a0d8eab163afe45b9d606df0e3bd Mon Sep 17 00:00:00 2001 From: Matzz Date: Mon, 14 Jul 2014 15:44:39 +0200 Subject: [PATCH 04/29] added some test + fixes --- .../queues/MySQLBasedDelayQueue.java | 16 +----- .../queues/MySQLBasedDelayQueueTests.java | 57 ++++++++++++++++--- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java index 6c543b0..0d5838a 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java @@ -42,7 +42,7 @@ public class MySQLBasedDelayQueue extends MySQLBasedQueue final static protected String delayedAddQuery = "INSERT INTO queue (queue_name, inserted, inserted_by, delayed_to, value) values (?, now(), ?, DATE_ADD(NOW(), INTERVAL ? SECOND), ?)"; - final static protected String delayedPeekQuery = "SELECT value FROM queue WHERE acquired IS NULL AND (delayed_to<=now() OR delayed_to is null) AND queue_name = ? ORDER BY id ASC LIMIT 1"; + final static protected String delayedPeekQuery = "SELECT value FROM queue WHERE acquired IS NULL AND (delayed_to<=NOW() OR delayed_to is null) AND queue_name = ? ORDER BY id ASC LIMIT 1"; final static String delayedPollQuery[] = { "SET @update_id := -1; ", "UPDATE queue SET " + @@ -51,7 +51,7 @@ public class MySQLBasedDelayQueue extends MySQLBasedQueue " acquired_by = ? " + "WHERE "+ "acquired IS NULL AND " + - "(delayed_to<=now() OR delayed_to is null) AND "+ + "(delayed_to<=NOW() OR delayed_to is null) AND "+ "queue_name = ? " + "ORDER BY id ASC " + "LIMIT 1; ", @@ -110,16 +110,6 @@ protected String getPeekQuery() { @Override protected String[] getPollQuery() { - return pollQuery; - } - - @Override - protected String getCleanupQuery() { - return cleanupQuery; - } - - @Override - protected String getCleanupAllQuery() { - return cleanupAllQuery; + return delayedPollQuery; } } diff --git a/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java b/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java index db228b0..b86bcbf 100644 --- a/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java +++ b/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java @@ -78,6 +78,35 @@ public void test() throws IOException { assertEmpty(); } + @Test + public void nonBlockingPeekTest() throws IOException, InterruptedException { + assertEmpty(); + long s = 2; + DelayedString a = new DelayedString("A", s); + assertTrue( queue.add(a) ); + assertEquals("Queue should contain one item", 1, queue.size()); + assertNull("Queue head should be null", queue.peek()); + Thread.sleep(s*2*1000l); + assertEquals("Queue head should be null", a, queue.peek()); + } + + @Test(timeout=10*1000) + public void delayedPollBlockingTest() throws IOException, InterruptedException, SQLException { + assertEmpty(); + long s = 2; + DelayedString a = new DelayedString("A", s); + + assertTrue( queue.add(a) ); + + assertNull("Queue head should be null", queue.peek()); + + Thread.sleep(s*2*1000l); + DelayedString ds = queue.poll(); + + assertEquals("Queue head should be null", a, ds); + assertEmpty(); + } + @Test(timeout=1000) public void pollBlockingTest() throws InterruptedException { assertEmpty(); @@ -85,7 +114,7 @@ public void pollBlockingTest() throws InterruptedException { long wait = WAIT_FOR_TIMING_TEST; long now = System.currentTimeMillis(); - String ret = queue.poll(wait, TimeUnit.MILLISECONDS).get(); + DelayedString ret = queue.poll(wait, TimeUnit.MILLISECONDS); long duration = System.currentTimeMillis() - now; assertNull("poll timed out", ret); @@ -102,11 +131,15 @@ protected static class DelayedString implements Delayed, Serializable { private String str; private long time; - private transient TimeUnit unit = TimeUnit.NANOSECONDS; + private transient TimeUnit unit = TimeUnit.SECONDS; public DelayedString(String str, long seconds) { this.str = str; - this.time = seconds + System.nanoTime(); + this.time = seconds + nowInSeconds(); + } + + public String get() { + return str; } @Override @@ -114,8 +147,15 @@ public int compareTo(Delayed o) { Long l = o.getDelay(unit); return l.compareTo(this.time); } + - public boolean equalsTo(Delayed o) { + @Override + public long getDelay(TimeUnit unit) { + return unit.convert(time - nowInSeconds(), unit); + } + + @Override + public boolean equals(Object o) { if(o instanceof DelayedString) { DelayedString v = (DelayedString) o; return get().equals(v.get()); @@ -125,14 +165,13 @@ public boolean equalsTo(Delayed o) { } } - @Override - public long getDelay(TimeUnit unit) { - return unit.convert(time - System.nanoTime(), unit); + public int hashCode() { + return get().hashCode(); } - public String get() { - return str; + private long nowInSeconds() { + return System.currentTimeMillis()/1000; } } } From 6725ea2542f4b191b83fe2a5a93f8e7478ce0a65 Mon Sep 17 00:00:00 2001 From: Matzz Date: Tue, 15 Jul 2014 10:47:10 +0200 Subject: [PATCH 05/29] major refactor --- .../bramp/concurrent/ConcurrentBitSet.java | 173 ---- .../java/net/bramp/concurrent/Futures.java | 104 --- .../locks/MySQLSleepBasedCondition.java | 228 ----- .../queues/AbstractBlockingQueue.java | 152 --- .../queues/MySQLBasedDelayQueue.java | 115 --- .../db_patterns/queues/MySQLBasedQueue.java | 340 ------- .../serializator/DefaultSerializator.java | 58 -- .../net/bramp/serializator/Serializator.java | 6 - .../java/net/bramp/sql/ResultSetFilter.java | 876 ------------------ src/main/java/net/bramp/sql/ResultSets.java | 21 - .../queues/MySQLBasedDelayQueueTests.java | 41 +- 11 files changed, 21 insertions(+), 2093 deletions(-) delete mode 100644 src/main/java/net/bramp/concurrent/ConcurrentBitSet.java delete mode 100644 src/main/java/net/bramp/concurrent/Futures.java delete mode 100644 src/main/java/net/bramp/db_patterns/locks/MySQLSleepBasedCondition.java delete mode 100644 src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java delete mode 100644 src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java delete mode 100644 src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java delete mode 100644 src/main/java/net/bramp/serializator/DefaultSerializator.java delete mode 100644 src/main/java/net/bramp/serializator/Serializator.java delete mode 100644 src/main/java/net/bramp/sql/ResultSetFilter.java delete mode 100644 src/main/java/net/bramp/sql/ResultSets.java diff --git a/src/main/java/net/bramp/concurrent/ConcurrentBitSet.java b/src/main/java/net/bramp/concurrent/ConcurrentBitSet.java deleted file mode 100644 index 28805e4..0000000 --- a/src/main/java/net/bramp/concurrent/ConcurrentBitSet.java +++ /dev/null @@ -1,173 +0,0 @@ -package net.bramp.concurrent; - -import java.util.BitSet; -import java.util.Set; -import java.util.TreeSet; - -/** - * Simple wrapper around BitSet to make it thread safe - * @author bramp - * - */ -public class ConcurrentBitSet { - final BitSet set; - - public ConcurrentBitSet() { - set = new BitSet(); - } - - public ConcurrentBitSet(int nbits) { - set = new BitSet(nbits); - } - - /** - * Atomically sets the value to the given updated value if the current value == the expected value. - * - * @param bitIndex - * @param expect - * @param update - * @return true if successful. False return indicates that the actual value was not equal to the expected value. - */ - public synchronized boolean compareAndSet(int bitIndex, boolean expect, boolean update) { - if (get(bitIndex) == expect) { - set(bitIndex, update); - return true; - } - return false; - } - - /** - * Return a set of clear indexes - * @return - */ - public Set getClearBits(int max) { - Set missing = new TreeSet(); - int nextBit = -1; - while (true) { - nextBit = this.nextClearBit(nextBit + 1); - if (nextBit >= max) - break; - missing.add(nextBit); - } - return missing; - } - - public synchronized byte[] toByteArray() { - return set.toByteArray(); - } - - public synchronized long[] toLongArray() { - return set.toLongArray(); - } - - public synchronized void flip(int bitIndex) { - set.flip(bitIndex); - } - - public synchronized void flip(int fromIndex, int toIndex) { - set.flip(fromIndex, toIndex); - } - - public synchronized void set(int bitIndex) { - set.set(bitIndex); - } - - public synchronized void set(int bitIndex, boolean value) { - set.set(bitIndex, value); - } - - public synchronized void set(int fromIndex, int toIndex) { - set.set(fromIndex, toIndex); - } - - public synchronized void set(int fromIndex, int toIndex, boolean value) { - set.set(fromIndex, toIndex, value); - } - - public synchronized void clear(int bitIndex) { - set.clear(bitIndex); - } - - public synchronized void clear(int fromIndex, int toIndex) { - set.clear(fromIndex, toIndex); - } - - public synchronized void clear() { - set.clear(); - } - - public synchronized boolean get(int bitIndex) { - return set.get(bitIndex); - } - - public synchronized BitSet get(int fromIndex, int toIndex) { - return set.get(fromIndex, toIndex); - } - - public synchronized int nextSetBit(int fromIndex) { - return set.nextSetBit(fromIndex); - } - - public synchronized int nextClearBit(int fromIndex) { - return set.nextClearBit(fromIndex); - } - - public synchronized int previousSetBit(int fromIndex) { - return set.previousSetBit(fromIndex); - } - - public synchronized int previousClearBit(int fromIndex) { - return set.previousClearBit(fromIndex); - } - - public synchronized int length() { - return set.length(); - } - - public synchronized boolean isEmpty() { - return set.isEmpty(); - } - - public synchronized boolean intersects(BitSet set) { - return set.intersects(set); - } - - public synchronized int cardinality() { - return set.cardinality(); - } - - public synchronized void and(BitSet set) { - set.and(set); - } - - public synchronized void or(BitSet set) { - set.or(set); - } - - public synchronized void xor(BitSet set) { - set.xor(set); - } - - public synchronized void andNot(BitSet set) { - set.andNot(set); - } - - public synchronized int hashCode() { - return set.hashCode(); - } - - public synchronized int size() { - return set.size(); - } - - public synchronized boolean equals(Object obj) { - if (obj instanceof ConcurrentBitSet) - return set.equals( ((ConcurrentBitSet)obj).set ); - - return false; - } - - public synchronized String toString() { - return set.toString(); - } -} \ No newline at end of file diff --git a/src/main/java/net/bramp/concurrent/Futures.java b/src/main/java/net/bramp/concurrent/Futures.java deleted file mode 100644 index 5d910e4..0000000 --- a/src/main/java/net/bramp/concurrent/Futures.java +++ /dev/null @@ -1,104 +0,0 @@ -package net.bramp.concurrent; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.BitSet; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -public final class Futures { - private Futures() {} - - /** - * Calls get() on all futures waiting for responses If any future throws an - * Exception, this method throws it (without returning the list of - * successful responses) - * - * @param futures - * @return - * @throws InterruptedException - * @throws ExecutionException - */ - public static List getAll(List> futures) - throws InterruptedException, ExecutionException { - List results = new ArrayList(futures.size()); - for (Future future : futures) { - results.add(future.get()); - } - return Collections.unmodifiableList(results); - } - - /** - * Calls get() on all futures waiting for responses until the timeout If any - * future throws an Exception, this method throws it (without returning the - * list of successful responses) - * - * @param futures - * @param timeout - * @param unit - * @return - * @throws InterruptedException - * @throws ExecutionException - * @throws TimeoutException - */ - @SuppressWarnings("unchecked") - public static List getAll(List> futures, long timeout, - TimeUnit unit) throws InterruptedException, ExecutionException, - TimeoutException { - - long timeoutNS = unit.toNanos(timeout); - long deadline = System.nanoTime() + timeoutNS; - - // Make a copy - futures = new LinkedList>(futures); - - final int size = futures.size(); - BitSet done = new BitSet(size); - - V[] results = (V[]) new Object[size]; - while (done.cardinality() < size) { - - for (int i = 0; i < size; i++) { - // TODO - We could use done.nextClearBit(i) - if (done.get(i)) - continue; - - Future future = futures.get(i); - try { - // We wait just a fraction, to give everyone at least two chances - results[i] = future.get(timeoutNS / (2 * size), TimeUnit.NANOSECONDS); - done.set(i); - - } catch (ExecutionException e) { - unwrapExecutionException(e); - - } catch (TimeoutException e) { - // If we have exceeded our deadline, throw, otherwise - if (System.nanoTime() >= deadline) - throw e; - } - } - } - - return Collections.unmodifiableList(Arrays.asList(results)); - } - - /** - * Sometimes InterruptedException is wrapped in an ExecutionException - * I don't think that's correct behavior, but lets fix it here - * @param e - * @throws InterruptedException - * @throws ExecutionException - */ - public static void unwrapExecutionException(ExecutionException e) throws InterruptedException, ExecutionException { - if (e.getCause() instanceof InterruptedException) - throw (InterruptedException)e.getCause(); - - throw e; - } -} diff --git a/src/main/java/net/bramp/db_patterns/locks/MySQLSleepBasedCondition.java b/src/main/java/net/bramp/db_patterns/locks/MySQLSleepBasedCondition.java deleted file mode 100644 index f366930..0000000 --- a/src/main/java/net/bramp/db_patterns/locks/MySQLSleepBasedCondition.java +++ /dev/null @@ -1,228 +0,0 @@ -package net.bramp.db_patterns.locks; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Condition; - -import javax.annotation.Nonnull; -import javax.sql.DataSource; - -import net.bramp.sql.ResultSetFilter; -import net.bramp.sql.ResultSets; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Uses the MySQL sleep() / kill to implement a distributed Condition - * - * @author bramp - */ -public class MySQLSleepBasedCondition implements Condition { - - final static Logger LOG = LoggerFactory.getLogger(MySQLSleepBasedCondition.class); - - static long DEFAULT_WAIT = 60000000000L; - - final static String sleepQuery = "SELECT SLEEP(?), ?;"; - final static String wakeQuery = "KILL QUERY ?;"; - - final static String listQueryNew = // MySQL 5.1.7 or newer - "SELECT Id, User, Host, Db, Command, Time, State, Info FROM " + - "INFORMATION_SCHEMA.PROCESSLIST " + - "WHERE STATE = 'User sleep' AND INFO LIKE ? " + - "ORDER BY TIME"; - - final static String listQueryOld = "SHOW PROCESSLIST;"; - - final boolean useListQueryNew = false; - - final DataSource ds; - final String lockName; - - final ResultSetFilter.Predicate isOurLockPredicate = new ResultSetFilter.Predicate() { - - public boolean apply(ResultSet rs) throws SQLException { - if (LOG.isDebugEnabled()) { - LOG.debug("ResultSet {}", ResultSets.toString(rs)); - } - - String state = rs.getString(7); - if (state != null && !state.equals("User sleep")) - return false; - - String info = rs.getString(8); - return info != null && info.matches("SELECT SLEEP\\([\\d.]+\\), '" + lockName + "'"); - } - }; - - public MySQLSleepBasedCondition(@Nonnull DataSource ds, @Nonnull String lockName) { - this.ds = ds; - this.lockName = lockName; - - // TODO Detect MySQL version (update useListQueryNew) - // TODO Detect if we can sleep/kill - } - - /** - * @param nanosTimeout The number of nanoseconds to wait - * @return true if awaken (correctly, or spuriously), false if timeout - * @throws InterruptedException - */ - protected boolean awaitNanosInternal(long nanosTimeout) throws InterruptedException { - if (nanosTimeout <= 0) - return false; - - long now = System.nanoTime(); - - try { - Connection c = ds.getConnection(); - try { - PreparedStatement s = c.prepareStatement(sleepQuery); - try { - - // Adjust nanosTimeout (due to time it took to get a connection) - nanosTimeout -= (System.nanoTime() - now); - - // Convert to seconds, but round to whole number of milliseconds - s.setFloat(1, Math.round(nanosTimeout / 1000000.0) / 1000f); - s.setString(2, lockName); - s.execute(); - - ResultSet rs = s.getResultSet(); - if (rs != null && rs.next()) - return rs.getInt(1) == 1; - - return true; - - } finally { - s.close(); - } - - } finally { - c.close(); - } - - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - public long awaitNanos(long nanosTimeout) throws InterruptedException { - long now = System.nanoTime(); - awaitNanosInternal(nanosTimeout); - return System.nanoTime() - now; - } - - public void await() throws InterruptedException { - while (!awaitNanosInternal(DEFAULT_WAIT)) { - // Keep looping, until we expire before our timeout or are interuptted - if (Thread.interrupted()) - throw new InterruptedException(); - - // TODO There is a race condition here. Between iterations we might miss a wakeup - } - } - - public void awaitUninterruptibly() { - while (true) { - try { - await(); - break; - } catch (InterruptedException e) { - } - } - } - - public boolean await(long time, TimeUnit unit) throws InterruptedException { - return awaitNanosInternal(unit.toNanos(time)); - } - - public boolean awaitUntil(Date deadline) throws InterruptedException { - long duration = deadline.getTime() - System.currentTimeMillis(); - return awaitNanosInternal(TimeUnit.MILLISECONDS.toNanos(duration)); - } - - /** - * Get a list of the other threads waiting - * - * @throws SQLException - */ - protected ResultSet findLockThreads(@Nonnull Connection c) throws SQLException { - PreparedStatement s = null; - - if (useListQueryNew) { - s = c.prepareStatement(listQueryNew); - s.setString(1, "SELECT SLEEP(%" + lockName + "%"); - } else { - s = c.prepareStatement(listQueryOld); - } - return new ResultSetFilter(s.executeQuery(), isOurLockPredicate); - } - - protected void killThread(@Nonnull Connection c, long threadId) throws SQLException { - LOG.debug("Killing thread {}", threadId); - - PreparedStatement s = c.prepareStatement(wakeQuery); - s.setLong(1, threadId); - s.execute(); - } - - /** - * Will signal the thread that's been waiting the longest - */ - public void signal() { - try { - Connection c = ds.getConnection(); - try { - // Find a list of blocked threads to wake up - ResultSet threads = findLockThreads(c); - if (!threads.next()) { - LOG.debug("Nothing to wake up for '{}'", lockName); - return; - } - long toWake = threads.getLong(1); - threads.close(); - - killThread(c, toWake); - - } finally { - c.close(); - } - - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - public void signalAll() { - try { - Connection c = ds.getConnection(); - try { - // Find a list of blocked threads to wake up - List toWake = new ArrayList(); - ResultSet threads = findLockThreads(c); - while (threads.next()) { - toWake.add(threads.getLong(1)); - } - threads.close(); - - for (Long id : toWake) { - killThread(c, id); - } - - } finally { - c.close(); - } - - } catch (SQLException e) { - throw new RuntimeException(e); - } - } -} diff --git a/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java b/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java deleted file mode 100644 index 40d97f2..0000000 --- a/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java +++ /dev/null @@ -1,152 +0,0 @@ -package net.bramp.db_patterns.queues; - -import java.util.Collection; -import java.util.Iterator; -import java.util.NoSuchElementException; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.TimeUnit; - -/** - * To keep code neat, most of the simple methods are here - * @author bramp - * - */ -abstract class AbstractBlockingQueue implements BlockingQueue { - - public boolean isEmpty() { - return size() == 0; - } - - public boolean offer(E e) { - return add(e); - } - - public E element() { - E head = peek(); - if (head == null) - throw new NoSuchElementException(); - return head; - } - - public E remove() { - E head = poll(); - if (head == null) - throw new NoSuchElementException(); - return head; - } - - /** - * Blocks - */ - public E take() throws InterruptedException { - // We loop around trying to get a item, blocking at most a minute at - // a time this allows us to be interrupted - E head = null; - while (head == null) { - if (Thread.interrupted()) - throw new InterruptedException(); - - head = poll(1, TimeUnit.MINUTES); - } - return head; - } - - /** - * Blocks - */ - public void put(E e) throws InterruptedException { - add(e); - } - - /** - * No blocking - */ - public int drainTo(Collection c) { - return drainTo(c, Integer.MAX_VALUE); - } - - /** - * No blocking - */ - public int drainTo(Collection c, int maxElements) { - if (c == this) - throw new IllegalArgumentException("Draining to self is not supported"); - - int count = 0; - while (count < maxElements) { - E head = poll(); - if (head == null) - break; - - c.add(head); - count++; - } - - return maxElements - count; - } - - public void clear() { - // Lazy! just keep poll'ng them off - while (poll() != null) { - // Nothing - } - } - - public int remainingCapacity() { - return Integer.MAX_VALUE; - } - - public boolean offer(E e, long timeout, TimeUnit unit) throws InterruptedException { - // Right now, we have no concept of a full queue, so we don't block on insert - return offer(e); - } - - ////// Nothing supported below - - public boolean contains(Object o) { - throw new UnsupportedOperationException(); - } - - public Iterator iterator() { - throw new UnsupportedOperationException(); - } - - public Object[] toArray() { - throw new UnsupportedOperationException(); } - - public T[] toArray(T[] a) { - throw new UnsupportedOperationException(); - } - - public boolean remove(Object o) { - throw new UnsupportedOperationException(); - } - - public boolean containsAll(Collection c) { - throw new UnsupportedOperationException(); - } - - public boolean addAll(Collection c) { - throw new UnsupportedOperationException(); - } - - public boolean removeAll(Collection c) { - throw new UnsupportedOperationException(); - } - - public boolean retainAll(Collection c) { - throw new UnsupportedOperationException(); - } - - protected abstract String getAddQuery(); - - protected abstract String getPeekQuery(); - - protected abstract String[] getPollQuery(); - - protected abstract String getSizeQuery(); - - protected abstract String getCleanupQuery(); - - protected abstract String getCleanupAllQuery(); -} diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java deleted file mode 100644 index 0d5838a..0000000 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java +++ /dev/null @@ -1,115 +0,0 @@ -package net.bramp.db_patterns.queues; - -import java.sql.CallableStatement; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Date; -import java.util.concurrent.Delayed; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Condition; - -import javax.sql.DataSource; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import net.bramp.db_patterns.locks.MySQLSleepBasedCondition; -import net.bramp.serializator.Serializator; - -/** - * A queue backed by MySQL - *

- * CREATE TABLE queue ( - * id INT UNSIGNED NOT NULL AUTO_INCREMENT, - * queue_name VARCHAR(255) NOT NULL, -- Queue name - * inserted TIMESTAMP NOT NULL, -- Time the row was inserted - * inserted_by VARCHAR(255) NOT NULL, -- and by who - * acquired TIMESTAMP NULL, -- Time the row was acquired - * acquired_by VARCHAR(255) NULL, -- and by who - * delayed_to TIMESTAMP NULL, - * value BLOB NOT NULL, -- The actual data - * PRIMARY KEY (id) - * ) ENGINE=INNODB DEFAULT CHARSET=UTF8; - *

- * TODO Create efficient drainTo - * - * @param - * @author bramp - */ -public class MySQLBasedDelayQueue extends MySQLBasedQueue { - - - final static protected String delayedAddQuery = "INSERT INTO queue (queue_name, inserted, inserted_by, delayed_to, value) values (?, now(), ?, DATE_ADD(NOW(), INTERVAL ? SECOND), ?)"; - final static protected String delayedPeekQuery = "SELECT value FROM queue WHERE acquired IS NULL AND (delayed_to<=NOW() OR delayed_to is null) AND queue_name = ? ORDER BY id ASC LIMIT 1"; - final static String delayedPollQuery[] = { - "SET @update_id := -1; ", - "UPDATE queue SET " + - " id = (SELECT @update_id := id), " + - " acquired = NOW(), " + - " acquired_by = ? " + - "WHERE "+ - "acquired IS NULL AND " + - "(delayed_to<=NOW() OR delayed_to is null) AND "+ - "queue_name = ? " + - "ORDER BY id ASC " + - "LIMIT 1; ", - "SELECT value FROM queue WHERE id = @update_id" - }; - - public MySQLBasedDelayQueue(DataSource ds, String queueName, Class type, String me) { - super(ds, queueName, type, me); - } - - public MySQLBasedDelayQueue(DataSource ds, String queueName, Serializator serializator, String me) { - super(ds, queueName, serializator, me); - } - - public boolean add(E value) { - try { - Connection c = ds.getConnection(); - try { - PreparedStatement s = c.prepareStatement(getAddQuery()); - try { - - - s.setString(1, queueName); - s.setObject(2, me); // Inserted by me - s.setLong(3, value.getDelay(TimeUnit.SECONDS)); - setValueToStatment(s, 4, value); - s.execute(); - - // Wake up one - condition.signal(); - - return true; - - } finally { - s.close(); - } - } finally { - c.close(); - } - - } catch (SQLException e) { - e.printStackTrace(); - throw new RuntimeException(e); - } - } - - @Override - protected String getAddQuery() { - return delayedAddQuery; - } - - @Override - protected String getPeekQuery() { - return delayedPeekQuery; - } - - @Override - protected String[] getPollQuery() { - return delayedPollQuery; - } -} diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java deleted file mode 100644 index 249ec6c..0000000 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java +++ /dev/null @@ -1,340 +0,0 @@ -package net.bramp.db_patterns.queues; - -import java.io.IOException; -import java.sql.CallableStatement; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Date; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Condition; - -import javax.sql.DataSource; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import net.bramp.db_patterns.locks.MySQLSleepBasedCondition; -import net.bramp.serializator.Serializator; - -/** - * A queue backed by MySQL - *

- * CREATE TABLE queue ( - * id INT UNSIGNED NOT NULL AUTO_INCREMENT, - * queue_name VARCHAR(255) NOT NULL, -- Queue name - * inserted TIMESTAMP NOT NULL, -- Time the row was inserted - * inserted_by VARCHAR(255) NOT NULL, -- and by who - * acquired TIMESTAMP NULL, -- Time the row was acquired - * acquired_by VARCHAR(255) NULL, -- and by who - * value BLOB NOT NULL, -- The actual data - * PRIMARY KEY (id) - * ) ENGINE=INNODB DEFAULT CHARSET=UTF8; - *

- * TODO Create efficient drainTo - * - * @param - * @author bramp - */ -public class MySQLBasedQueue extends AbstractBlockingQueue { - - final static Logger LOG = LoggerFactory.getLogger(MySQLBasedQueue.class); - - final static String addQuery = "INSERT INTO queue (queue_name, inserted, inserted_by, value) values (?, now(), ?, ?)"; - final static String peekQuery = "SELECT value FROM queue WHERE acquired IS NULL AND queue_name = ? ORDER BY id ASC LIMIT 1"; - final static String sizeQuery = "SELECT COUNT(*) FROM queue WHERE acquired IS NULL AND queue_name = ?"; - - /** - * Claims one row (and keeps it in the database) - */ - final static String pollQuery[] = { - "SET @update_id := -1; ", - - "UPDATE queue SET " + " id = (SELECT @update_id := id), " - + " acquired = NOW(), " + " acquired_by = ? " - + "WHERE acquired IS NULL AND queue_name = ? " - + "ORDER BY id ASC " + "LIMIT 1; ", - - "SELECT value FROM queue WHERE id = @update_id" - }; - - final static String cleanupQuery = "DELETE FROM queue " - + "WHERE acquired IS NOT NULL " + " AND queue_name = ? " - + " AND acquired < DATE_SUB(NOW(), INTERVAL 10 DAY)"; - - final static String cleanupAllQuery = "DELETE FROM queue " - + "WHERE acquired IS NOT NULL " - + " AND acquired < DATE_SUB(NOW(), INTERVAL 10 DAY)"; - - final String me; - - final DataSource ds; - final String queueName; - - protected Class type = null; - protected Serializator serializator = null; - - final Condition condition; - - /** - * Creates a new MySQL backed queue - * - * @param ds - * @param queueName - * @param type - * @param me The name of this node, for storing in the database table - */ - public MySQLBasedQueue(DataSource ds, String queueName, Class type, String me) { - this(ds, queueName, me); - this.type = type; - } - - /** - * Creates a new MySQL backed queue - * - * @param ds - * @param queueName - * @param serializator - * @param me The name of this node, for storing in the database table - */ - public MySQLBasedQueue(DataSource ds, String queueName, Serializator serializator, String me) { - this(ds, queueName, me); - this.serializator = serializator; - } - - protected MySQLBasedQueue(DataSource ds, String queueName, String me) { - this.ds = ds; - this.queueName = queueName; - this.condition = new MySQLSleepBasedCondition(ds, "queue-" + queueName); - this.me = me; - } - - public boolean add(E value) { - try { - Connection c = ds.getConnection(); - try { - PreparedStatement s = c.prepareStatement(getAddQuery()); - try { - s.setString(1, queueName); - s.setObject(2, me); // Inserted by me - setValueToStatment(s, 3, value); - s.execute(); - - // Wake up one - condition.signal(); - - return true; - - } finally { - s.close(); - } - } finally { - c.close(); - } - - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - /** - * No blocking - */ - public E peek() { - try { - Connection c = ds.getConnection(); - try { - PreparedStatement s = c.prepareStatement(getPeekQuery()); - try { - s.setString(1, queueName); - if (s.execute()) { - ResultSet rs = s.getResultSet(); - if (rs != null && rs.next()) { - return getValueFromResult(rs, 1); - } - } - - return null; - } finally { - s.close(); - } - - } finally { - c.close(); - } - - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - /** - * No blocking - */ - public E poll() { - try { - Connection c = ds.getConnection(); - String[] pollQuery = getPollQuery(); - try { - c.setAutoCommit(false); - - CallableStatement s1 = c.prepareCall(pollQuery[0]); - s1.execute(); - - PreparedStatement s2 = c.prepareStatement(pollQuery[1]); - s2.setString(1, me); // Acquired by me - s2.setString(2, queueName); - s2.execute(); - - CallableStatement s3 = c.prepareCall(pollQuery[2]); - s3.execute(); - - c.commit(); - - if (s3.execute()) { - ResultSet rs = s3.getResultSet(); - if (rs != null && rs.next()) { - return getValueFromResult(rs, 1); - } - } - - return null; - - } finally { - c.setAutoCommit(true); - c.close(); - } - - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - public int size() { - try { - Connection c = ds.getConnection(); - try { - PreparedStatement s = c.prepareStatement(getSizeQuery()); - s.setString(1, queueName); - s.execute(); - - ResultSet rs = s.getResultSet(); - if (rs != null && rs.next()) - return rs.getInt(1); - - throw new RuntimeException("Failed to retreive size"); - - } finally { - c.close(); - } - - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - /** - * Blocks until something is in the queue, up to timeout null if timeout - * occurs - */ - public E poll(long timeout, TimeUnit unit) throws InterruptedException { - - final long deadlineMillis = System.currentTimeMillis() + unit.toMillis(timeout); - final Date deadline = new Date(deadlineMillis); - - E head = null; - boolean stillWaiting = true; - - while (stillWaiting) { - // Check if we can grab one - head = poll(); - if (head != null) - break; - - // Block until we are woken, or deadline - // Because we don't have a distributed lock around this condition, there is a race condition - // whereby we might miss a notify(). However, we can somewhat mitigate the problem, by using - // this in a polling fashion - stillWaiting = condition.awaitUntil(deadline); - } - - return head; - } - - public void cleanup() throws SQLException { - Connection c = ds.getConnection(); - try { - CallableStatement s = c.prepareCall(getCleanupAllQuery()); - s.setString(1, queueName); - s.execute(); - - } finally { - c.close(); - } - } - - /** - * Cleans up all queues - * - * @throws SQLException - */ - public void cleanupAll() throws SQLException { - Connection c = ds.getConnection(); - try { - CallableStatement s = c.prepareCall(getCleanupAllQuery()); - s.execute(); - - } finally { - c.close(); - } - } - - @Override - protected String getAddQuery() { - return addQuery; - } - - @Override - protected String getPeekQuery() { - return peekQuery; - } - - @Override - protected String[] getPollQuery() { - return pollQuery; - } - - @Override - protected String getSizeQuery() { - return sizeQuery; - } - - @Override - protected String getCleanupQuery() { - return cleanupQuery; - } - - @Override - protected String getCleanupAllQuery() { - return cleanupAllQuery; - } - - protected E getValueFromResult(ResultSet rs, int index) throws SQLException { - if(serializator == null) { - return rs.getObject(1, type); - } - else { - return serializator.deserialize(rs.getBytes(index)); - } - } - - protected void setValueToStatment(PreparedStatement s, int index, E obj) throws SQLException { - if(serializator == null) { - s.setObject(index, obj); - } - else { - s.setBytes(index, serializator.serialize(obj)); - } - } -} diff --git a/src/main/java/net/bramp/serializator/DefaultSerializator.java b/src/main/java/net/bramp/serializator/DefaultSerializator.java deleted file mode 100644 index 8d109f9..0000000 --- a/src/main/java/net/bramp/serializator/DefaultSerializator.java +++ /dev/null @@ -1,58 +0,0 @@ -package net.bramp.serializator; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; - -public class DefaultSerializator implements Serializator { - - @Override - public byte[] serialize(E obj) { - byte[] array = null; - ByteArrayOutputStream out = new ByteArrayOutputStream(); - try { - try { - ObjectOutputStream objectOut = new ObjectOutputStream(out); - try { - objectOut.writeObject(obj); - array = out.toByteArray(); - } catch (IOException e) { - objectOut.close(); - } - } finally { - out.close(); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - - return array; - } - - @Override - public E deserialize(byte[] bytes) { - E obj = null; - ByteArrayInputStream in = new ByteArrayInputStream(bytes); - try { - try { - ObjectInputStream objectIn = new ObjectInputStream(in); - try { - obj = (E) objectIn.readObject(); - } catch (IOException e) { - objectIn.close(); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - } finally { - in.close(); - } - - } catch (IOException e) { - throw new RuntimeException(e); - } - return obj; - } - -} diff --git a/src/main/java/net/bramp/serializator/Serializator.java b/src/main/java/net/bramp/serializator/Serializator.java deleted file mode 100644 index d328861..0000000 --- a/src/main/java/net/bramp/serializator/Serializator.java +++ /dev/null @@ -1,6 +0,0 @@ -package net.bramp.serializator; - -public interface Serializator { - public byte[] serialize(E obj); - public E deserialize(byte[] bytes); -} diff --git a/src/main/java/net/bramp/sql/ResultSetFilter.java b/src/main/java/net/bramp/sql/ResultSetFilter.java deleted file mode 100644 index 48ec247..0000000 --- a/src/main/java/net/bramp/sql/ResultSetFilter.java +++ /dev/null @@ -1,876 +0,0 @@ -package net.bramp.sql; - -import java.io.InputStream; -import java.io.Reader; -import java.math.BigDecimal; -import java.net.URL; -import java.sql.Array; -import java.sql.Blob; -import java.sql.Clob; -import java.sql.Date; -import java.sql.NClob; -import java.sql.Ref; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.RowId; -import java.sql.SQLException; -import java.sql.SQLWarning; -import java.sql.SQLXML; -import java.sql.Statement; -import java.sql.Time; -import java.sql.Timestamp; -import java.util.Calendar; -import java.util.Map; - -import javax.annotation.Nonnull; - -/** - * Wraps an existing ResultSet, and filters out rows that don't match the predicate - * - * @author bramp - * - */ -public class ResultSetFilter implements ResultSet { - final ResultSet rs; - final Predicate predicate; - - public interface Predicate { - /** - * Return true if this row is acceptable - * Return false if it is not - * Do not call next() - * @param rs - * @return - * @throws SQLException - */ - public boolean apply(@Nonnull ResultSet rs) throws SQLException; - } - - public ResultSetFilter(@Nonnull ResultSet rs, @Nonnull Predicate predicate) { - this.rs = rs; - this.predicate = predicate; - } - - public T unwrap(Class iface) throws SQLException { - return rs.unwrap(iface); - } - - public boolean isWrapperFor(Class iface) throws SQLException { - return rs.isWrapperFor(iface); - } - - public boolean next() throws SQLException { - while(rs.next()) { - if (predicate.apply(rs)) - return true; - } - return false; - } - - public void close() throws SQLException { - rs.close(); - } - - public boolean wasNull() throws SQLException { - return rs.wasNull(); - } - - public String getString(int columnIndex) throws SQLException { - return rs.getString(columnIndex); - } - - public boolean getBoolean(int columnIndex) throws SQLException { - return rs.getBoolean(columnIndex); - } - - public byte getByte(int columnIndex) throws SQLException { - return rs.getByte(columnIndex); - } - - public short getShort(int columnIndex) throws SQLException { - return rs.getShort(columnIndex); - } - - public int getInt(int columnIndex) throws SQLException { - return rs.getInt(columnIndex); - } - - public long getLong(int columnIndex) throws SQLException { - return rs.getLong(columnIndex); - } - - public float getFloat(int columnIndex) throws SQLException { - return rs.getFloat(columnIndex); - } - - public double getDouble(int columnIndex) throws SQLException { - return rs.getDouble(columnIndex); - } - - @Deprecated - public BigDecimal getBigDecimal(int columnIndex, int scale) - throws SQLException { - return rs.getBigDecimal(columnIndex, scale); - } - - public byte[] getBytes(int columnIndex) throws SQLException { - return rs.getBytes(columnIndex); - } - - public Date getDate(int columnIndex) throws SQLException { - return rs.getDate(columnIndex); - } - - public Time getTime(int columnIndex) throws SQLException { - return rs.getTime(columnIndex); - } - - public Timestamp getTimestamp(int columnIndex) throws SQLException { - return rs.getTimestamp(columnIndex); - } - - public InputStream getAsciiStream(int columnIndex) throws SQLException { - return rs.getAsciiStream(columnIndex); - } - - @Deprecated - public InputStream getUnicodeStream(int columnIndex) throws SQLException { - return rs.getUnicodeStream(columnIndex); - } - - public InputStream getBinaryStream(int columnIndex) throws SQLException { - return rs.getBinaryStream(columnIndex); - } - - public String getString(String columnLabel) throws SQLException { - return rs.getString(columnLabel); - } - - public boolean getBoolean(String columnLabel) throws SQLException { - return rs.getBoolean(columnLabel); - } - - public byte getByte(String columnLabel) throws SQLException { - return rs.getByte(columnLabel); - } - - public short getShort(String columnLabel) throws SQLException { - return rs.getShort(columnLabel); - } - - public int getInt(String columnLabel) throws SQLException { - return rs.getInt(columnLabel); - } - - public long getLong(String columnLabel) throws SQLException { - return rs.getLong(columnLabel); - } - - public float getFloat(String columnLabel) throws SQLException { - return rs.getFloat(columnLabel); - } - - public double getDouble(String columnLabel) throws SQLException { - return rs.getDouble(columnLabel); - } - - @Deprecated - public BigDecimal getBigDecimal(String columnLabel, int scale) - throws SQLException { - return rs.getBigDecimal(columnLabel, scale); - } - - public byte[] getBytes(String columnLabel) throws SQLException { - return rs.getBytes(columnLabel); - } - - public Date getDate(String columnLabel) throws SQLException { - return rs.getDate(columnLabel); - } - - public Time getTime(String columnLabel) throws SQLException { - return rs.getTime(columnLabel); - } - - public Timestamp getTimestamp(String columnLabel) throws SQLException { - return rs.getTimestamp(columnLabel); - } - - public InputStream getAsciiStream(String columnLabel) throws SQLException { - return rs.getAsciiStream(columnLabel); - } - - @Deprecated - public InputStream getUnicodeStream(String columnLabel) throws SQLException { - return rs.getUnicodeStream(columnLabel); - } - - public InputStream getBinaryStream(String columnLabel) throws SQLException { - return rs.getBinaryStream(columnLabel); - } - - public SQLWarning getWarnings() throws SQLException { - return rs.getWarnings(); - } - - public void clearWarnings() throws SQLException { - rs.clearWarnings(); - } - - public String getCursorName() throws SQLException { - return rs.getCursorName(); - } - - public ResultSetMetaData getMetaData() throws SQLException { - return rs.getMetaData(); - } - - public Object getObject(int columnIndex) throws SQLException { - return rs.getObject(columnIndex); - } - - public Object getObject(String columnLabel) throws SQLException { - return rs.getObject(columnLabel); - } - - public int findColumn(String columnLabel) throws SQLException { - return rs.findColumn(columnLabel); - } - - public Reader getCharacterStream(int columnIndex) throws SQLException { - return rs.getCharacterStream(columnIndex); - } - - public Reader getCharacterStream(String columnLabel) throws SQLException { - return rs.getCharacterStream(columnLabel); - } - - public BigDecimal getBigDecimal(int columnIndex) throws SQLException { - return rs.getBigDecimal(columnIndex); - } - - public BigDecimal getBigDecimal(String columnLabel) throws SQLException { - return rs.getBigDecimal(columnLabel); - } - - public boolean isBeforeFirst() throws SQLException { - return rs.isBeforeFirst(); - } - - public boolean isAfterLast() throws SQLException { - return rs.isAfterLast(); - } - - public boolean isFirst() throws SQLException { - return rs.isFirst(); - } - - public boolean isLast() throws SQLException { - return rs.isLast(); - } - - public void beforeFirst() throws SQLException { - rs.beforeFirst(); - } - - public void afterLast() throws SQLException { - rs.afterLast(); - } - - public boolean first() throws SQLException { - return rs.first(); - } - - public boolean last() throws SQLException { - return rs.last(); - } - - public int getRow() throws SQLException { - return rs.getRow(); - } - - public boolean absolute(int row) throws SQLException { - return rs.absolute(row); - } - - public boolean relative(int rows) throws SQLException { - return rs.relative(rows); - } - - public boolean previous() throws SQLException { - return rs.previous(); - } - - public void setFetchDirection(int direction) throws SQLException { - rs.setFetchDirection(direction); - } - - public int getFetchDirection() throws SQLException { - return rs.getFetchDirection(); - } - - public void setFetchSize(int rows) throws SQLException { - rs.setFetchSize(rows); - } - - public int getFetchSize() throws SQLException { - return rs.getFetchSize(); - } - - public int getType() throws SQLException { - return rs.getType(); - } - - public int getConcurrency() throws SQLException { - return rs.getConcurrency(); - } - - public boolean rowUpdated() throws SQLException { - return rs.rowUpdated(); - } - - public boolean rowInserted() throws SQLException { - return rs.rowInserted(); - } - - public boolean rowDeleted() throws SQLException { - return rs.rowDeleted(); - } - - public void updateNull(int columnIndex) throws SQLException { - rs.updateNull(columnIndex); - } - - public void updateBoolean(int columnIndex, boolean x) throws SQLException { - rs.updateBoolean(columnIndex, x); - } - - public void updateByte(int columnIndex, byte x) throws SQLException { - rs.updateByte(columnIndex, x); - } - - public void updateShort(int columnIndex, short x) throws SQLException { - rs.updateShort(columnIndex, x); - } - - public void updateInt(int columnIndex, int x) throws SQLException { - rs.updateInt(columnIndex, x); - } - - public void updateLong(int columnIndex, long x) throws SQLException { - rs.updateLong(columnIndex, x); - } - - public void updateFloat(int columnIndex, float x) throws SQLException { - rs.updateFloat(columnIndex, x); - } - - public void updateDouble(int columnIndex, double x) throws SQLException { - rs.updateDouble(columnIndex, x); - } - - public void updateBigDecimal(int columnIndex, BigDecimal x) - throws SQLException { - rs.updateBigDecimal(columnIndex, x); - } - - public void updateString(int columnIndex, String x) throws SQLException { - rs.updateString(columnIndex, x); - } - - public void updateBytes(int columnIndex, byte[] x) throws SQLException { - rs.updateBytes(columnIndex, x); - } - - public void updateDate(int columnIndex, Date x) throws SQLException { - rs.updateDate(columnIndex, x); - } - - public void updateTime(int columnIndex, Time x) throws SQLException { - rs.updateTime(columnIndex, x); - } - - public void updateTimestamp(int columnIndex, Timestamp x) - throws SQLException { - rs.updateTimestamp(columnIndex, x); - } - - public void updateAsciiStream(int columnIndex, InputStream x, int length) - throws SQLException { - rs.updateAsciiStream(columnIndex, x, length); - } - - public void updateBinaryStream(int columnIndex, InputStream x, int length) - throws SQLException { - rs.updateBinaryStream(columnIndex, x, length); - } - - public void updateCharacterStream(int columnIndex, Reader x, int length) - throws SQLException { - rs.updateCharacterStream(columnIndex, x, length); - } - - public void updateObject(int columnIndex, Object x, int scaleOrLength) - throws SQLException { - rs.updateObject(columnIndex, x, scaleOrLength); - } - - public void updateObject(int columnIndex, Object x) throws SQLException { - rs.updateObject(columnIndex, x); - } - - public void updateNull(String columnLabel) throws SQLException { - rs.updateNull(columnLabel); - } - - public void updateBoolean(String columnLabel, boolean x) - throws SQLException { - rs.updateBoolean(columnLabel, x); - } - - public void updateByte(String columnLabel, byte x) throws SQLException { - rs.updateByte(columnLabel, x); - } - - public void updateShort(String columnLabel, short x) throws SQLException { - rs.updateShort(columnLabel, x); - } - - public void updateInt(String columnLabel, int x) throws SQLException { - rs.updateInt(columnLabel, x); - } - - public void updateLong(String columnLabel, long x) throws SQLException { - rs.updateLong(columnLabel, x); - } - - public void updateFloat(String columnLabel, float x) throws SQLException { - rs.updateFloat(columnLabel, x); - } - - public void updateDouble(String columnLabel, double x) throws SQLException { - rs.updateDouble(columnLabel, x); - } - - public void updateBigDecimal(String columnLabel, BigDecimal x) - throws SQLException { - rs.updateBigDecimal(columnLabel, x); - } - - public void updateString(String columnLabel, String x) throws SQLException { - rs.updateString(columnLabel, x); - } - - public void updateBytes(String columnLabel, byte[] x) throws SQLException { - rs.updateBytes(columnLabel, x); - } - - public void updateDate(String columnLabel, Date x) throws SQLException { - rs.updateDate(columnLabel, x); - } - - public void updateTime(String columnLabel, Time x) throws SQLException { - rs.updateTime(columnLabel, x); - } - - public void updateTimestamp(String columnLabel, Timestamp x) - throws SQLException { - rs.updateTimestamp(columnLabel, x); - } - - public void updateAsciiStream(String columnLabel, InputStream x, int length) - throws SQLException { - rs.updateAsciiStream(columnLabel, x, length); - } - - public void updateBinaryStream(String columnLabel, InputStream x, int length) - throws SQLException { - rs.updateBinaryStream(columnLabel, x, length); - } - - public void updateCharacterStream(String columnLabel, Reader reader, - int length) throws SQLException { - rs.updateCharacterStream(columnLabel, reader, length); - } - - public void updateObject(String columnLabel, Object x, int scaleOrLength) - throws SQLException { - rs.updateObject(columnLabel, x, scaleOrLength); - } - - public void updateObject(String columnLabel, Object x) throws SQLException { - rs.updateObject(columnLabel, x); - } - - public void insertRow() throws SQLException { - rs.insertRow(); - } - - public void updateRow() throws SQLException { - rs.updateRow(); - } - - public void deleteRow() throws SQLException { - rs.deleteRow(); - } - - public void refreshRow() throws SQLException { - rs.refreshRow(); - } - - public void cancelRowUpdates() throws SQLException { - rs.cancelRowUpdates(); - } - - public void moveToInsertRow() throws SQLException { - rs.moveToInsertRow(); - } - - public void moveToCurrentRow() throws SQLException { - rs.moveToCurrentRow(); - } - - public Statement getStatement() throws SQLException { - return rs.getStatement(); - } - - public Object getObject(int columnIndex, Map> map) - throws SQLException { - return rs.getObject(columnIndex, map); - } - - public Ref getRef(int columnIndex) throws SQLException { - return rs.getRef(columnIndex); - } - - public Blob getBlob(int columnIndex) throws SQLException { - return rs.getBlob(columnIndex); - } - - public Clob getClob(int columnIndex) throws SQLException { - return rs.getClob(columnIndex); - } - - public Array getArray(int columnIndex) throws SQLException { - return rs.getArray(columnIndex); - } - - public Object getObject(String columnLabel, Map> map) - throws SQLException { - return rs.getObject(columnLabel, map); - } - - public Ref getRef(String columnLabel) throws SQLException { - return rs.getRef(columnLabel); - } - - public Blob getBlob(String columnLabel) throws SQLException { - return rs.getBlob(columnLabel); - } - - public Clob getClob(String columnLabel) throws SQLException { - return rs.getClob(columnLabel); - } - - public Array getArray(String columnLabel) throws SQLException { - return rs.getArray(columnLabel); - } - - public Date getDate(int columnIndex, Calendar cal) throws SQLException { - return rs.getDate(columnIndex, cal); - } - - public Date getDate(String columnLabel, Calendar cal) throws SQLException { - return rs.getDate(columnLabel, cal); - } - - public Time getTime(int columnIndex, Calendar cal) throws SQLException { - return rs.getTime(columnIndex, cal); - } - - public Time getTime(String columnLabel, Calendar cal) throws SQLException { - return rs.getTime(columnLabel, cal); - } - - public Timestamp getTimestamp(int columnIndex, Calendar cal) - throws SQLException { - return rs.getTimestamp(columnIndex, cal); - } - - public Timestamp getTimestamp(String columnLabel, Calendar cal) - throws SQLException { - return rs.getTimestamp(columnLabel, cal); - } - - public URL getURL(int columnIndex) throws SQLException { - return rs.getURL(columnIndex); - } - - public URL getURL(String columnLabel) throws SQLException { - return rs.getURL(columnLabel); - } - - public void updateRef(int columnIndex, Ref x) throws SQLException { - rs.updateRef(columnIndex, x); - } - - public void updateRef(String columnLabel, Ref x) throws SQLException { - rs.updateRef(columnLabel, x); - } - - public void updateBlob(int columnIndex, Blob x) throws SQLException { - rs.updateBlob(columnIndex, x); - } - - public void updateBlob(String columnLabel, Blob x) throws SQLException { - rs.updateBlob(columnLabel, x); - } - - public void updateClob(int columnIndex, Clob x) throws SQLException { - rs.updateClob(columnIndex, x); - } - - public void updateClob(String columnLabel, Clob x) throws SQLException { - rs.updateClob(columnLabel, x); - } - - public void updateArray(int columnIndex, Array x) throws SQLException { - rs.updateArray(columnIndex, x); - } - - public void updateArray(String columnLabel, Array x) throws SQLException { - rs.updateArray(columnLabel, x); - } - - public RowId getRowId(int columnIndex) throws SQLException { - return rs.getRowId(columnIndex); - } - - public RowId getRowId(String columnLabel) throws SQLException { - return rs.getRowId(columnLabel); - } - - public void updateRowId(int columnIndex, RowId x) throws SQLException { - rs.updateRowId(columnIndex, x); - } - - public void updateRowId(String columnLabel, RowId x) throws SQLException { - rs.updateRowId(columnLabel, x); - } - - public int getHoldability() throws SQLException { - return rs.getHoldability(); - } - - public boolean isClosed() throws SQLException { - return rs.isClosed(); - } - - public void updateNString(int columnIndex, String nString) - throws SQLException { - rs.updateNString(columnIndex, nString); - } - - public void updateNString(String columnLabel, String nString) - throws SQLException { - rs.updateNString(columnLabel, nString); - } - - public void updateNClob(int columnIndex, NClob nClob) throws SQLException { - rs.updateNClob(columnIndex, nClob); - } - - public void updateNClob(String columnLabel, NClob nClob) - throws SQLException { - rs.updateNClob(columnLabel, nClob); - } - - public NClob getNClob(int columnIndex) throws SQLException { - return rs.getNClob(columnIndex); - } - - public NClob getNClob(String columnLabel) throws SQLException { - return rs.getNClob(columnLabel); - } - - public SQLXML getSQLXML(int columnIndex) throws SQLException { - return rs.getSQLXML(columnIndex); - } - - public SQLXML getSQLXML(String columnLabel) throws SQLException { - return rs.getSQLXML(columnLabel); - } - - public void updateSQLXML(int columnIndex, SQLXML xmlObject) - throws SQLException { - rs.updateSQLXML(columnIndex, xmlObject); - } - - public void updateSQLXML(String columnLabel, SQLXML xmlObject) - throws SQLException { - rs.updateSQLXML(columnLabel, xmlObject); - } - - public String getNString(int columnIndex) throws SQLException { - return rs.getNString(columnIndex); - } - - public String getNString(String columnLabel) throws SQLException { - return rs.getNString(columnLabel); - } - - public Reader getNCharacterStream(int columnIndex) throws SQLException { - return rs.getNCharacterStream(columnIndex); - } - - public Reader getNCharacterStream(String columnLabel) throws SQLException { - return rs.getNCharacterStream(columnLabel); - } - - public void updateNCharacterStream(int columnIndex, Reader x, long length) - throws SQLException { - rs.updateNCharacterStream(columnIndex, x, length); - } - - public void updateNCharacterStream(String columnLabel, Reader reader, - long length) throws SQLException { - rs.updateNCharacterStream(columnLabel, reader, length); - } - - public void updateAsciiStream(int columnIndex, InputStream x, long length) - throws SQLException { - rs.updateAsciiStream(columnIndex, x, length); - } - - public void updateBinaryStream(int columnIndex, InputStream x, long length) - throws SQLException { - rs.updateBinaryStream(columnIndex, x, length); - } - - public void updateCharacterStream(int columnIndex, Reader x, long length) - throws SQLException { - rs.updateCharacterStream(columnIndex, x, length); - } - - public void updateAsciiStream(String columnLabel, InputStream x, long length) - throws SQLException { - rs.updateAsciiStream(columnLabel, x, length); - } - - public void updateBinaryStream(String columnLabel, InputStream x, - long length) throws SQLException { - rs.updateBinaryStream(columnLabel, x, length); - } - - public void updateCharacterStream(String columnLabel, Reader reader, - long length) throws SQLException { - rs.updateCharacterStream(columnLabel, reader, length); - } - - public void updateBlob(int columnIndex, InputStream inputStream, long length) - throws SQLException { - rs.updateBlob(columnIndex, inputStream, length); - } - - public void updateBlob(String columnLabel, InputStream inputStream, - long length) throws SQLException { - rs.updateBlob(columnLabel, inputStream, length); - } - - public void updateClob(int columnIndex, Reader reader, long length) - throws SQLException { - rs.updateClob(columnIndex, reader, length); - } - - public void updateClob(String columnLabel, Reader reader, long length) - throws SQLException { - rs.updateClob(columnLabel, reader, length); - } - - public void updateNClob(int columnIndex, Reader reader, long length) - throws SQLException { - rs.updateNClob(columnIndex, reader, length); - } - - public void updateNClob(String columnLabel, Reader reader, long length) - throws SQLException { - rs.updateNClob(columnLabel, reader, length); - } - - public void updateNCharacterStream(int columnIndex, Reader x) - throws SQLException { - rs.updateNCharacterStream(columnIndex, x); - } - - public void updateNCharacterStream(String columnLabel, Reader reader) - throws SQLException { - rs.updateNCharacterStream(columnLabel, reader); - } - - public void updateAsciiStream(int columnIndex, InputStream x) - throws SQLException { - rs.updateAsciiStream(columnIndex, x); - } - - public void updateBinaryStream(int columnIndex, InputStream x) - throws SQLException { - rs.updateBinaryStream(columnIndex, x); - } - - public void updateCharacterStream(int columnIndex, Reader x) - throws SQLException { - rs.updateCharacterStream(columnIndex, x); - } - - public void updateAsciiStream(String columnLabel, InputStream x) - throws SQLException { - rs.updateAsciiStream(columnLabel, x); - } - - public void updateBinaryStream(String columnLabel, InputStream x) - throws SQLException { - rs.updateBinaryStream(columnLabel, x); - } - - public void updateCharacterStream(String columnLabel, Reader reader) - throws SQLException { - rs.updateCharacterStream(columnLabel, reader); - } - - public void updateBlob(int columnIndex, InputStream inputStream) - throws SQLException { - rs.updateBlob(columnIndex, inputStream); - } - - public void updateBlob(String columnLabel, InputStream inputStream) - throws SQLException { - rs.updateBlob(columnLabel, inputStream); - } - - public void updateClob(int columnIndex, Reader reader) throws SQLException { - rs.updateClob(columnIndex, reader); - } - - public void updateClob(String columnLabel, Reader reader) - throws SQLException { - rs.updateClob(columnLabel, reader); - } - - public void updateNClob(int columnIndex, Reader reader) throws SQLException { - rs.updateNClob(columnIndex, reader); - } - - public void updateNClob(String columnLabel, Reader reader) - throws SQLException { - rs.updateNClob(columnLabel, reader); - } - - public T getObject(int columnIndex, Class type) throws SQLException { - return rs.getObject(columnIndex, type); - } - - public T getObject(String columnLabel, Class type) - throws SQLException { - return rs.getObject(columnLabel, type); - } -} diff --git a/src/main/java/net/bramp/sql/ResultSets.java b/src/main/java/net/bramp/sql/ResultSets.java deleted file mode 100644 index 33faf2b..0000000 --- a/src/main/java/net/bramp/sql/ResultSets.java +++ /dev/null @@ -1,21 +0,0 @@ -package net.bramp.sql; - -import java.sql.ResultSet; -import java.sql.SQLException; - -public final class ResultSets { - private ResultSets() {} - - public static String toString(ResultSet rs) throws SQLException { - StringBuilder sb = new StringBuilder(); - int cols = rs.getMetaData().getColumnCount(); - for (int i = 1; i <= cols; i++) { - sb.append('"').append( rs.getString(i) ).append('"').append( ", "); - } - - if (cols > 0) - sb.setLength( sb.length() - 2); - - return sb.toString(); - } -} diff --git a/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java b/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java index b86bcbf..7e3d534 100644 --- a/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java +++ b/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java @@ -22,25 +22,27 @@ public class MySQLBasedDelayQueueTests { final static long WAIT_FOR_TIMING_TEST = 300; // in ms - + + private String queueTable; private String queueName; private DataSource ds; - private MySQLBasedQueue queue; + private MySQLBasedDelayQueue queue; @Before public void setup() { // Different queue name for each test (to avoid test clashes) + queueTable = "queue"; queueName = java.util.UUID.randomUUID().toString(); ds = DatabaseUtils.createDataSource(); - queue = new MySQLBasedDelayQueue(ds, queueName, new DefaultSerializator(), "test"); + queue = new MySQLBasedDelayQueue(ds, queueTable, queueName, new DefaultSerializator(), "test"); } @After public void cleanupDatabase() throws SQLException { queue.clear(); - queue.cleanupAll(); + queue.cleanupAll(0); assertEmpty(); } @@ -90,22 +92,21 @@ public void nonBlockingPeekTest() throws IOException, InterruptedException { assertEquals("Queue head should be null", a, queue.peek()); } - @Test(timeout=10*1000) - public void delayedPollBlockingTest() throws IOException, InterruptedException, SQLException { - assertEmpty(); - long s = 2; - DelayedString a = new DelayedString("A", s); - - assertTrue( queue.add(a) ); - - assertNull("Queue head should be null", queue.peek()); - - Thread.sleep(s*2*1000l); - DelayedString ds = queue.poll(); - - assertEquals("Queue head should be null", a, ds); - assertEmpty(); - } +// @Test(timeout=10000) +// public void delayedPollBlockingTest() throws IOException, InterruptedException, SQLException { +// assertEmpty(); +// long s = 2; +// DelayedString a = new DelayedString("A", s); +// +// assertTrue( queue.add(a) ); +// assertNull("Queue head should be null", queue.peek()); +// +// +// DelayedString ds = queue.poll(s*2, TimeUnit.SECONDS); +// +// assertEquals("Queue head should be object", a, ds); +// assertEmpty(); +// } @Test(timeout=1000) public void pollBlockingTest() throws InterruptedException { From ca160c3c5706551561ab76f7b127aa666b8171e5 Mon Sep 17 00:00:00 2001 From: Matzz Date: Tue, 15 Jul 2014 10:59:14 +0200 Subject: [PATCH 06/29] test database config in separate file --- .../net/bramp/db_patterns/DatabaseUtils.java | 33 ++++++++++++++++--- src/test/resources/database.properties | 4 +++ 2 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 src/test/resources/database.properties diff --git a/src/test/java/net/bramp/db_patterns/DatabaseUtils.java b/src/test/java/net/bramp/db_patterns/DatabaseUtils.java index 29149cf..839e2f5 100644 --- a/src/test/java/net/bramp/db_patterns/DatabaseUtils.java +++ b/src/test/java/net/bramp/db_patterns/DatabaseUtils.java @@ -5,16 +5,25 @@ import com.google.common.base.Throwables; import com.mysql.jdbc.jdbc2.optional.MysqlDataSource; +import java.io.IOException; +import java.io.InputStream; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.Properties; public class DatabaseUtils { public static DataSource createDataSource() { MysqlDataSource ds = new MysqlDataSource(); - ds.setUser("root"); - ds.setPassword("sP6prUCe"); - ds.setServerName("localhost"); - ds.setDatabaseName("db_patterns"); + Properties prop; + try { + prop = getDatabaseProperties(); + ds.setUser(prop.getProperty("user")); + ds.setPassword(prop.getProperty("password")); + ds.setServerName(prop.getProperty("serverName")); + ds.setDatabaseName(prop.getProperty("databaseName")); + } catch (IOException e) { + e.printStackTrace(); + } return ds; } @@ -25,4 +34,20 @@ public static String getHostname() { throw Throwables.propagate(e); } } + + protected static Properties getDatabaseProperties() throws IOException { + Properties prop = new Properties(); + InputStream is = null; + try { + is = DatabaseUtils.class.getClassLoader().getResourceAsStream("database.properties"); + prop.load(is); + + } + finally { + if(is!=null) { + is.close(); + } + } + return prop; + } } diff --git a/src/test/resources/database.properties b/src/test/resources/database.properties new file mode 100644 index 0000000..1f0d517 --- /dev/null +++ b/src/test/resources/database.properties @@ -0,0 +1,4 @@ +user=root +password= +serverName=localhost +databaseName=db_patterns \ No newline at end of file From b7cc06035429f788c0c82e2399aa75bfc629ff00 Mon Sep 17 00:00:00 2001 From: Matzz Date: Tue, 15 Jul 2014 11:37:26 +0200 Subject: [PATCH 07/29] test fixes --- .../bramp/concurrent/ConcurrentBitSet.java | 173 ++++ .../java/net/bramp/concurrent/Futures.java | 104 +++ .../locks/MySQLSleepBasedCondition.java | 228 +++++ .../queues/AbstractBlockingQueue.java | 139 +++ .../queues/AbstractMySQLQueue.java | 351 +++++++ .../queues/MySQLBasedDelayQueue.java | 93 ++ .../db_patterns/queues/MySQLBasedQueue.java | 90 ++ .../serializator/DefaultSerializator.java | 58 ++ .../net/bramp/serializator/Serializator.java | 6 + .../java/net/bramp/sql/ResultSetFilter.java | 876 ++++++++++++++++++ src/main/java/net/bramp/sql/ResultSets.java | 21 + .../queues/MySQLBasedQueueTests.java | 2 +- 12 files changed, 2140 insertions(+), 1 deletion(-) create mode 100644 src/main/java/net/bramp/concurrent/ConcurrentBitSet.java create mode 100644 src/main/java/net/bramp/concurrent/Futures.java create mode 100644 src/main/java/net/bramp/db_patterns/locks/MySQLSleepBasedCondition.java create mode 100644 src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java create mode 100644 src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java create mode 100644 src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java create mode 100644 src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java create mode 100644 src/main/java/net/bramp/serializator/DefaultSerializator.java create mode 100644 src/main/java/net/bramp/serializator/Serializator.java create mode 100644 src/main/java/net/bramp/sql/ResultSetFilter.java create mode 100644 src/main/java/net/bramp/sql/ResultSets.java diff --git a/src/main/java/net/bramp/concurrent/ConcurrentBitSet.java b/src/main/java/net/bramp/concurrent/ConcurrentBitSet.java new file mode 100644 index 0000000..28805e4 --- /dev/null +++ b/src/main/java/net/bramp/concurrent/ConcurrentBitSet.java @@ -0,0 +1,173 @@ +package net.bramp.concurrent; + +import java.util.BitSet; +import java.util.Set; +import java.util.TreeSet; + +/** + * Simple wrapper around BitSet to make it thread safe + * @author bramp + * + */ +public class ConcurrentBitSet { + final BitSet set; + + public ConcurrentBitSet() { + set = new BitSet(); + } + + public ConcurrentBitSet(int nbits) { + set = new BitSet(nbits); + } + + /** + * Atomically sets the value to the given updated value if the current value == the expected value. + * + * @param bitIndex + * @param expect + * @param update + * @return true if successful. False return indicates that the actual value was not equal to the expected value. + */ + public synchronized boolean compareAndSet(int bitIndex, boolean expect, boolean update) { + if (get(bitIndex) == expect) { + set(bitIndex, update); + return true; + } + return false; + } + + /** + * Return a set of clear indexes + * @return + */ + public Set getClearBits(int max) { + Set missing = new TreeSet(); + int nextBit = -1; + while (true) { + nextBit = this.nextClearBit(nextBit + 1); + if (nextBit >= max) + break; + missing.add(nextBit); + } + return missing; + } + + public synchronized byte[] toByteArray() { + return set.toByteArray(); + } + + public synchronized long[] toLongArray() { + return set.toLongArray(); + } + + public synchronized void flip(int bitIndex) { + set.flip(bitIndex); + } + + public synchronized void flip(int fromIndex, int toIndex) { + set.flip(fromIndex, toIndex); + } + + public synchronized void set(int bitIndex) { + set.set(bitIndex); + } + + public synchronized void set(int bitIndex, boolean value) { + set.set(bitIndex, value); + } + + public synchronized void set(int fromIndex, int toIndex) { + set.set(fromIndex, toIndex); + } + + public synchronized void set(int fromIndex, int toIndex, boolean value) { + set.set(fromIndex, toIndex, value); + } + + public synchronized void clear(int bitIndex) { + set.clear(bitIndex); + } + + public synchronized void clear(int fromIndex, int toIndex) { + set.clear(fromIndex, toIndex); + } + + public synchronized void clear() { + set.clear(); + } + + public synchronized boolean get(int bitIndex) { + return set.get(bitIndex); + } + + public synchronized BitSet get(int fromIndex, int toIndex) { + return set.get(fromIndex, toIndex); + } + + public synchronized int nextSetBit(int fromIndex) { + return set.nextSetBit(fromIndex); + } + + public synchronized int nextClearBit(int fromIndex) { + return set.nextClearBit(fromIndex); + } + + public synchronized int previousSetBit(int fromIndex) { + return set.previousSetBit(fromIndex); + } + + public synchronized int previousClearBit(int fromIndex) { + return set.previousClearBit(fromIndex); + } + + public synchronized int length() { + return set.length(); + } + + public synchronized boolean isEmpty() { + return set.isEmpty(); + } + + public synchronized boolean intersects(BitSet set) { + return set.intersects(set); + } + + public synchronized int cardinality() { + return set.cardinality(); + } + + public synchronized void and(BitSet set) { + set.and(set); + } + + public synchronized void or(BitSet set) { + set.or(set); + } + + public synchronized void xor(BitSet set) { + set.xor(set); + } + + public synchronized void andNot(BitSet set) { + set.andNot(set); + } + + public synchronized int hashCode() { + return set.hashCode(); + } + + public synchronized int size() { + return set.size(); + } + + public synchronized boolean equals(Object obj) { + if (obj instanceof ConcurrentBitSet) + return set.equals( ((ConcurrentBitSet)obj).set ); + + return false; + } + + public synchronized String toString() { + return set.toString(); + } +} \ No newline at end of file diff --git a/src/main/java/net/bramp/concurrent/Futures.java b/src/main/java/net/bramp/concurrent/Futures.java new file mode 100644 index 0000000..5d910e4 --- /dev/null +++ b/src/main/java/net/bramp/concurrent/Futures.java @@ -0,0 +1,104 @@ +package net.bramp.concurrent; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public final class Futures { + private Futures() {} + + /** + * Calls get() on all futures waiting for responses If any future throws an + * Exception, this method throws it (without returning the list of + * successful responses) + * + * @param futures + * @return + * @throws InterruptedException + * @throws ExecutionException + */ + public static List getAll(List> futures) + throws InterruptedException, ExecutionException { + List results = new ArrayList(futures.size()); + for (Future future : futures) { + results.add(future.get()); + } + return Collections.unmodifiableList(results); + } + + /** + * Calls get() on all futures waiting for responses until the timeout If any + * future throws an Exception, this method throws it (without returning the + * list of successful responses) + * + * @param futures + * @param timeout + * @param unit + * @return + * @throws InterruptedException + * @throws ExecutionException + * @throws TimeoutException + */ + @SuppressWarnings("unchecked") + public static List getAll(List> futures, long timeout, + TimeUnit unit) throws InterruptedException, ExecutionException, + TimeoutException { + + long timeoutNS = unit.toNanos(timeout); + long deadline = System.nanoTime() + timeoutNS; + + // Make a copy + futures = new LinkedList>(futures); + + final int size = futures.size(); + BitSet done = new BitSet(size); + + V[] results = (V[]) new Object[size]; + while (done.cardinality() < size) { + + for (int i = 0; i < size; i++) { + // TODO - We could use done.nextClearBit(i) + if (done.get(i)) + continue; + + Future future = futures.get(i); + try { + // We wait just a fraction, to give everyone at least two chances + results[i] = future.get(timeoutNS / (2 * size), TimeUnit.NANOSECONDS); + done.set(i); + + } catch (ExecutionException e) { + unwrapExecutionException(e); + + } catch (TimeoutException e) { + // If we have exceeded our deadline, throw, otherwise + if (System.nanoTime() >= deadline) + throw e; + } + } + } + + return Collections.unmodifiableList(Arrays.asList(results)); + } + + /** + * Sometimes InterruptedException is wrapped in an ExecutionException + * I don't think that's correct behavior, but lets fix it here + * @param e + * @throws InterruptedException + * @throws ExecutionException + */ + public static void unwrapExecutionException(ExecutionException e) throws InterruptedException, ExecutionException { + if (e.getCause() instanceof InterruptedException) + throw (InterruptedException)e.getCause(); + + throw e; + } +} diff --git a/src/main/java/net/bramp/db_patterns/locks/MySQLSleepBasedCondition.java b/src/main/java/net/bramp/db_patterns/locks/MySQLSleepBasedCondition.java new file mode 100644 index 0000000..f366930 --- /dev/null +++ b/src/main/java/net/bramp/db_patterns/locks/MySQLSleepBasedCondition.java @@ -0,0 +1,228 @@ +package net.bramp.db_patterns.locks; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; + +import javax.annotation.Nonnull; +import javax.sql.DataSource; + +import net.bramp.sql.ResultSetFilter; +import net.bramp.sql.ResultSets; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Uses the MySQL sleep() / kill to implement a distributed Condition + * + * @author bramp + */ +public class MySQLSleepBasedCondition implements Condition { + + final static Logger LOG = LoggerFactory.getLogger(MySQLSleepBasedCondition.class); + + static long DEFAULT_WAIT = 60000000000L; + + final static String sleepQuery = "SELECT SLEEP(?), ?;"; + final static String wakeQuery = "KILL QUERY ?;"; + + final static String listQueryNew = // MySQL 5.1.7 or newer + "SELECT Id, User, Host, Db, Command, Time, State, Info FROM " + + "INFORMATION_SCHEMA.PROCESSLIST " + + "WHERE STATE = 'User sleep' AND INFO LIKE ? " + + "ORDER BY TIME"; + + final static String listQueryOld = "SHOW PROCESSLIST;"; + + final boolean useListQueryNew = false; + + final DataSource ds; + final String lockName; + + final ResultSetFilter.Predicate isOurLockPredicate = new ResultSetFilter.Predicate() { + + public boolean apply(ResultSet rs) throws SQLException { + if (LOG.isDebugEnabled()) { + LOG.debug("ResultSet {}", ResultSets.toString(rs)); + } + + String state = rs.getString(7); + if (state != null && !state.equals("User sleep")) + return false; + + String info = rs.getString(8); + return info != null && info.matches("SELECT SLEEP\\([\\d.]+\\), '" + lockName + "'"); + } + }; + + public MySQLSleepBasedCondition(@Nonnull DataSource ds, @Nonnull String lockName) { + this.ds = ds; + this.lockName = lockName; + + // TODO Detect MySQL version (update useListQueryNew) + // TODO Detect if we can sleep/kill + } + + /** + * @param nanosTimeout The number of nanoseconds to wait + * @return true if awaken (correctly, or spuriously), false if timeout + * @throws InterruptedException + */ + protected boolean awaitNanosInternal(long nanosTimeout) throws InterruptedException { + if (nanosTimeout <= 0) + return false; + + long now = System.nanoTime(); + + try { + Connection c = ds.getConnection(); + try { + PreparedStatement s = c.prepareStatement(sleepQuery); + try { + + // Adjust nanosTimeout (due to time it took to get a connection) + nanosTimeout -= (System.nanoTime() - now); + + // Convert to seconds, but round to whole number of milliseconds + s.setFloat(1, Math.round(nanosTimeout / 1000000.0) / 1000f); + s.setString(2, lockName); + s.execute(); + + ResultSet rs = s.getResultSet(); + if (rs != null && rs.next()) + return rs.getInt(1) == 1; + + return true; + + } finally { + s.close(); + } + + } finally { + c.close(); + } + + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public long awaitNanos(long nanosTimeout) throws InterruptedException { + long now = System.nanoTime(); + awaitNanosInternal(nanosTimeout); + return System.nanoTime() - now; + } + + public void await() throws InterruptedException { + while (!awaitNanosInternal(DEFAULT_WAIT)) { + // Keep looping, until we expire before our timeout or are interuptted + if (Thread.interrupted()) + throw new InterruptedException(); + + // TODO There is a race condition here. Between iterations we might miss a wakeup + } + } + + public void awaitUninterruptibly() { + while (true) { + try { + await(); + break; + } catch (InterruptedException e) { + } + } + } + + public boolean await(long time, TimeUnit unit) throws InterruptedException { + return awaitNanosInternal(unit.toNanos(time)); + } + + public boolean awaitUntil(Date deadline) throws InterruptedException { + long duration = deadline.getTime() - System.currentTimeMillis(); + return awaitNanosInternal(TimeUnit.MILLISECONDS.toNanos(duration)); + } + + /** + * Get a list of the other threads waiting + * + * @throws SQLException + */ + protected ResultSet findLockThreads(@Nonnull Connection c) throws SQLException { + PreparedStatement s = null; + + if (useListQueryNew) { + s = c.prepareStatement(listQueryNew); + s.setString(1, "SELECT SLEEP(%" + lockName + "%"); + } else { + s = c.prepareStatement(listQueryOld); + } + return new ResultSetFilter(s.executeQuery(), isOurLockPredicate); + } + + protected void killThread(@Nonnull Connection c, long threadId) throws SQLException { + LOG.debug("Killing thread {}", threadId); + + PreparedStatement s = c.prepareStatement(wakeQuery); + s.setLong(1, threadId); + s.execute(); + } + + /** + * Will signal the thread that's been waiting the longest + */ + public void signal() { + try { + Connection c = ds.getConnection(); + try { + // Find a list of blocked threads to wake up + ResultSet threads = findLockThreads(c); + if (!threads.next()) { + LOG.debug("Nothing to wake up for '{}'", lockName); + return; + } + long toWake = threads.getLong(1); + threads.close(); + + killThread(c, toWake); + + } finally { + c.close(); + } + + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public void signalAll() { + try { + Connection c = ds.getConnection(); + try { + // Find a list of blocked threads to wake up + List toWake = new ArrayList(); + ResultSet threads = findLockThreads(c); + while (threads.next()) { + toWake.add(threads.getLong(1)); + } + threads.close(); + + for (Long id : toWake) { + killThread(c, id); + } + + } finally { + c.close(); + } + + } catch (SQLException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java b/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java new file mode 100644 index 0000000..2c6d2f9 --- /dev/null +++ b/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java @@ -0,0 +1,139 @@ +package net.bramp.db_patterns.queues; + +import java.util.Collection; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; + +/** + * To keep code neat, most of the simple methods are here + * @author bramp + */ +abstract class AbstractBlockingQueue implements BlockingQueue { + + public boolean isEmpty() { + return size() == 0; + } + + public boolean offer(E e) { + return add(e); + } + + public E element() { + E head = peek(); + if (head == null) + throw new NoSuchElementException(); + return head; + } + + public E remove() { + E head = poll(); + if (head == null) + throw new NoSuchElementException(); + return head; + } + + /** + * Blocks + */ + public E take() throws InterruptedException { + // We loop around trying to get a item, blocking at most a minute at + // a time this allows us to be interrupted + E head = null; + while (head == null) { + if (Thread.interrupted()) + throw new InterruptedException(); + + head = poll(1, TimeUnit.MINUTES); + } + return head; + } + + /** + * Blocks + */ + public void put(E e) throws InterruptedException { + add(e); + } + + /** + * No blocking + */ + public int drainTo(Collection c) { + return drainTo(c, Integer.MAX_VALUE); + } + + /** + * No blocking + */ + public int drainTo(Collection c, int maxElements) { + if (c == this) + throw new IllegalArgumentException("Draining to self is not supported"); + + int count = 0; + while (count < maxElements) { + E head = poll(); + if (head == null) + break; + + c.add(head); + count++; + } + + return maxElements - count; + } + + public void clear() { + // Lazy! just keep poll'ng them off + while (poll() != null) { + // Nothing + } + } + + public int remainingCapacity() { + return Integer.MAX_VALUE; + } + + public boolean offer(E e, long timeout, TimeUnit unit) throws InterruptedException { + // Right now, we have no concept of a full queue, so we don't block on insert + return offer(e); + } + + ////// Nothing supported below + + public boolean contains(Object o) { + throw new UnsupportedOperationException(); + } + + public Iterator iterator() { + throw new UnsupportedOperationException(); + } + + public Object[] toArray() { + throw new UnsupportedOperationException(); } + + public T[] toArray(T[] a) { + throw new UnsupportedOperationException(); + } + + public boolean remove(Object o) { + throw new UnsupportedOperationException(); + } + + public boolean containsAll(Collection c) { + throw new UnsupportedOperationException(); + } + + public boolean addAll(Collection c) { + throw new UnsupportedOperationException(); + } + + public boolean removeAll(Collection c) { + throw new UnsupportedOperationException(); + } + + public boolean retainAll(Collection c) { + throw new UnsupportedOperationException(); + } +} diff --git a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java new file mode 100644 index 0000000..56dfc6c --- /dev/null +++ b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java @@ -0,0 +1,351 @@ +package net.bramp.db_patterns.queues; + +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Date; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; + +import javax.sql.DataSource; + +import net.bramp.db_patterns.locks.MySQLSleepBasedCondition; +import net.bramp.serializator.Serializator; + +/** + * TODO Create efficient drainTo + * + * @param + * @author bramp + */ +abstract class AbstractMySQLQueue extends AbstractBlockingQueue { + + protected String me; + protected DataSource ds; + protected String queueName; + protected String tableName; + + protected Class type = null; + protected Serializator serializator = null; + protected Condition condition; + + final static String tableNamePlaceholder = "%TABLE_NAME%"; + protected String addQuery; + protected String peekQuery; + protected String[] pollQuery; + + protected String cleanupQuery = "DELETE FROM " + tableNamePlaceholder + + " " + "WHERE acquired IS NOT NULL " + " AND queue_name = ? " + + " AND acquired < DATE_SUB(NOW(), INTERVAL ? DAY)"; + + protected String cleanupAllQuery = "DELETE FROM " + tableNamePlaceholder + + " " + "WHERE acquired IS NOT NULL " + + " AND acquired < DATE_SUB(NOW(), INTERVAL ? DAY)"; + + protected String sizeQuery = "SELECT COUNT(*) FROM queue WHERE acquired IS NULL AND queue_name = ?"; + + /** + * Creates a new MySQL backed queue. Store values using statement setObject. + * + * @param ds + * @param queueTableName + * @param queueName + * @param type + * @param me + * The name of this node, for storing in the database table + */ + public AbstractMySQLQueue(DataSource ds, String queueTableName, + String queueName, Class type, String me) { + this(ds, queueTableName, queueName, me); + this.type = type; + } + + /** + * Creates a new MySQL backed queue. Store values using serializator and + * setBytes. + * + * @param ds + * @param queueName + * @param serializator + * @param me + * The name of this node, for storing in the database table + */ + public AbstractMySQLQueue(DataSource ds, String queueTableName, + String queueName, Serializator serializator, String me) { + this(ds, queueTableName, queueName, me); + this.serializator = serializator; + } + + protected AbstractMySQLQueue(DataSource ds, String tableName, + String queueName, String me) { + this.ds = ds; + this.tableName = escapeTableName(tableName); + this.queueName = queueName; + this.condition = new MySQLSleepBasedCondition(ds, "queue-" + queueName); + this.me = me; + } + + public boolean add(E value) { + try { + Connection c = ds.getConnection(); + try { + PreparedStatement s = c.prepareStatement(getAddQuery()); + try { + setAddParameters(value, s); + s.execute(); + wakeupThread(); + return true; + + } finally { + s.close(); + } + } finally { + c.close(); + } + + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + /** + * No blocking + */ + public E peek() { + try { + Connection c = ds.getConnection(); + try { + PreparedStatement s = c.prepareStatement(getPeekQuery()); + try { + s.setString(1, queueName); + if (s.execute()) { + ResultSet rs = s.getResultSet(); + if (rs != null && rs.next()) { + return getValueFromResult(rs, 1); + } + } + + return null; + } finally { + s.close(); + } + + } finally { + c.close(); + } + + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + /** + * No blocking + */ + public E poll() { + try { + Connection c = ds.getConnection(); + String[] pollQuery = getPollQuery(); + try { + c.setAutoCommit(false); + + CallableStatement s1 = c.prepareCall(pollQuery[0]); + s1.execute(); + + PreparedStatement s2 = c.prepareStatement(pollQuery[1]); + s2.setString(1, me); // Acquired by me + s2.setString(2, queueName); + s2.execute(); + + CallableStatement s3 = c.prepareCall(pollQuery[2]); + s3.execute(); + + c.commit(); + + if (s3.execute()) { + ResultSet rs = s3.getResultSet(); + if (rs != null && rs.next()) { + return getValueFromResult(rs, 1); + } + } + + return null; + + } finally { + c.setAutoCommit(true); + c.close(); + } + + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + public int size() { + try { + Connection c = ds.getConnection(); + try { + PreparedStatement s = c.prepareStatement(getSizeQuery()); + s.setString(1, queueName); + s.execute(); + + ResultSet rs = s.getResultSet(); + if (rs != null && rs.next()) + return rs.getInt(1); + + throw new RuntimeException("Failed to retreive size"); + + } finally { + c.close(); + } + + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + /** + * Blocks until something is in the queue, up to timeout null if timeout + * occurs + */ + public E poll(long timeout, TimeUnit unit) throws InterruptedException { + + final long deadlineMillis = System.currentTimeMillis() + + unit.toMillis(timeout); + final Date deadline = new Date(deadlineMillis); + + E head = null; + boolean stillWaiting = true; + + while (stillWaiting) { + // Check if we can grab one + head = poll(); + if (head != null) + break; + + // Block until we are woken, or deadline + // Because we don't have a distributed lock around this condition, + // there is a race condition + // whereby we might miss a notify(). However, we can somewhat + // mitigate the problem, by using + // this in a polling fashion + stillWaiting = condition.awaitUntil(deadline); + } + + return head; + } + + /** + * Legacy cleanupAll + * + * @deprecated + * @throws SQLException + */ + public void cleanup() throws SQLException { + cleanup(10); + } + + public void cleanup(int days) throws SQLException { + Connection c = ds.getConnection(); + try { + CallableStatement s = c.prepareCall(getCleanupAllQuery()); + s.setString(1, queueName); + s.setInt(2, days); + s.execute(); + + } finally { + c.close(); + } + } + + /** + * Legacy cleanupAll + * + * @deprecated + * @throws SQLException + */ + public void cleanupAll() throws SQLException { + cleanupAll(10); + } + + /** + * Cleans up all queues + * + * @throws SQLException + */ + public void cleanupAll(int days) throws SQLException { + Connection c = ds.getConnection(); + try { + CallableStatement s = c.prepareCall(getCleanupAllQuery()); + s.setInt(1, days); + s.execute(); + + } finally { + c.close(); + } + } + + protected void setAddParameters(E value, PreparedStatement s) throws SQLException { + } + + protected String setTable(String query) { + return query.replaceAll(tableNamePlaceholder, tableName); + } + + protected String escapeTableName(String tableName) { + return "`" + tableName.replaceAll("`", "") + "`"; + } + + protected void wakeupThread() { + condition.signal(); + } + + protected E getValueFromResult(ResultSet rs, int index) throws SQLException { + if (serializator == null) { + return rs.getObject(1, type); + } else { + return serializator.deserialize(rs.getBytes(index)); + } + } + + protected void setValueToStatment(PreparedStatement s, int index, E obj) + throws SQLException { + if (serializator == null) { + s.setObject(index, obj); + } else { + s.setBytes(index, serializator.serialize(obj)); + } + } + + + protected String getAddQuery() { + return setTable(addQuery); + } + + protected String getPeekQuery() { + return setTable(peekQuery); + } + + protected String[] getPollQuery() { + String[] queries = new String[pollQuery.length]; + for(int i=0; i + * @author matzz + */ +public class MySQLBasedDelayQueue extends + AbstractMySQLQueue { + + protected String closestDelayQuery = "SELECT min(delayed_to)-NOW() FROM " + tableNamePlaceholder + + " WHERE acquired IS NULL AND queue_name = ?"; + + protected String delayCondition = "AND (delayed_to<=NOW() OR delayed_to is null) "; + { + addQuery = "INSERT INTO "+tableNamePlaceholder+" " + + "(queue_name, inserted, inserted_by, delayed_to, value) values " + + "(?, now(), ?, DATE_ADD(NOW(), INTERVAL ? SECOND), ?)"; + + peekQuery = "SELECT value FROM "+tableNamePlaceholder+" WHERE " + + "acquired IS NULL " + + delayCondition + + "AND queue_name = ? " + + "ORDER BY id ASC LIMIT 1"; + + pollQuery = new String[] { + "SET @update_id := -1; ", + "UPDATE "+tableNamePlaceholder+" SET " + + " id = (SELECT @update_id := id), " + + " acquired = NOW(), " + + " acquired_by = ? " + + "WHERE " + "acquired IS NULL " + delayCondition + + "AND queue_name = ? " + "ORDER BY id ASC " + "LIMIT 1; ", + "SELECT value FROM "+tableNamePlaceholder+" WHERE id = @update_id" + }; + } + + + /** + * {@inheritDoc} + */ + public MySQLBasedDelayQueue(DataSource ds, String queueTableName, + String queueName, Class type, String me) { + super(ds, queueTableName, queueName, type, me); + } + + /** + * {@inheritDoc} + */ + public MySQLBasedDelayQueue(DataSource ds, String queueTableName, + String queueName, Serializator serializator, String me) { + super(ds, queueTableName, queueName, serializator, me); + } + + @Override + protected void setAddParameters(E value, PreparedStatement s) + throws SQLException { + s.setString(1, queueName); + s.setObject(2, me); // Inserted by me + s.setLong(3, value.getDelay(TimeUnit.SECONDS)); + setValueToStatment(s, 4, value); + } + + @Override + protected void wakeupThread() { + condition.signal(); + } + + protected long getClosestDelay() { + return 1; + } + +} diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java new file mode 100644 index 0000000..f0e1b36 --- /dev/null +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java @@ -0,0 +1,90 @@ +package net.bramp.db_patterns.queues; + +import java.sql.PreparedStatement; +import java.sql.SQLException; + +import javax.sql.DataSource; + +import net.bramp.serializator.Serializator; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A queue backed by MySQL + *

+ * CREATE TABLE queue ( + * id INT UNSIGNED NOT NULL AUTO_INCREMENT, + * queue_name VARCHAR(255) NOT NULL, -- Queue name + * inserted TIMESTAMP NOT NULL, -- Time the row was inserted + * inserted_by VARCHAR(255) NOT NULL, -- and by who + * acquired TIMESTAMP NULL, -- Time the row was acquired + * acquired_by VARCHAR(255) NULL, -- and by who + * value BLOB NOT NULL, -- The actual data + * PRIMARY KEY (id) + * ) ENGINE=INNODB DEFAULT CHARSET=UTF8; + *

+ * TODO Create efficient drainTo + * + * @param + * @author bramp + */ +public class MySQLBasedQueue extends AbstractMySQLQueue { + + protected Logger LOG = LoggerFactory.getLogger(MySQLBasedQueue.class); + + { + addQuery = "INSERT INTO "+tableNamePlaceholder+" " + + "(queue_name, inserted, inserted_by, value) values " + + "(?, now(), ?, ?)"; + peekQuery = "SELECT value FROM "+tableNamePlaceholder+" WHERE " + + "acquired IS NULL " + + "AND queue_name = ? " + + "ORDER BY id ASC LIMIT 1"; + pollQuery = new String[] { + "SET @update_id := -1; ", + "UPDATE "+tableNamePlaceholder+" SET " + + " id = (SELECT @update_id := id), " + + " acquired = NOW(), " + + " acquired_by = ? " + + "WHERE " + "acquired IS NULL " + + "AND queue_name = ? " + "ORDER BY id ASC " + "LIMIT 1; ", + "SELECT value FROM "+tableNamePlaceholder+" WHERE id = @update_id" + }; + } + + /** + * Legacy constructor + * + * @param ds + * @param queueName + * @param type + * @param me The name of this node, for storing in the database table + * @deprecated + */ + public MySQLBasedQueue(DataSource ds, String queueName, Class type, String me) { + super(ds, "queue", queueName, me); + this.type = type; + } + + @Override + protected void setAddParameters(E value, PreparedStatement s) throws SQLException { + s.setString(1, queueName); + s.setObject(2, me); // Inserted by me + setValueToStatment(s, 3, value); + } + + /** + * {@inheritDoc} + */ + public MySQLBasedQueue(DataSource ds, String queueTableName, String queueName, Class type, String me) { + super(ds, queueTableName, queueName, type, me); + } + + /** + * {@inheritDoc} + */ + public MySQLBasedQueue(DataSource ds, String queueTableName, String queueName, Serializator serializator, String me) { + super(ds, queueTableName, queueName, serializator, me); + } +} diff --git a/src/main/java/net/bramp/serializator/DefaultSerializator.java b/src/main/java/net/bramp/serializator/DefaultSerializator.java new file mode 100644 index 0000000..8d109f9 --- /dev/null +++ b/src/main/java/net/bramp/serializator/DefaultSerializator.java @@ -0,0 +1,58 @@ +package net.bramp.serializator; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + +public class DefaultSerializator implements Serializator { + + @Override + public byte[] serialize(E obj) { + byte[] array = null; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + try { + ObjectOutputStream objectOut = new ObjectOutputStream(out); + try { + objectOut.writeObject(obj); + array = out.toByteArray(); + } catch (IOException e) { + objectOut.close(); + } + } finally { + out.close(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + + return array; + } + + @Override + public E deserialize(byte[] bytes) { + E obj = null; + ByteArrayInputStream in = new ByteArrayInputStream(bytes); + try { + try { + ObjectInputStream objectIn = new ObjectInputStream(in); + try { + obj = (E) objectIn.readObject(); + } catch (IOException e) { + objectIn.close(); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } finally { + in.close(); + } + + } catch (IOException e) { + throw new RuntimeException(e); + } + return obj; + } + +} diff --git a/src/main/java/net/bramp/serializator/Serializator.java b/src/main/java/net/bramp/serializator/Serializator.java new file mode 100644 index 0000000..d328861 --- /dev/null +++ b/src/main/java/net/bramp/serializator/Serializator.java @@ -0,0 +1,6 @@ +package net.bramp.serializator; + +public interface Serializator { + public byte[] serialize(E obj); + public E deserialize(byte[] bytes); +} diff --git a/src/main/java/net/bramp/sql/ResultSetFilter.java b/src/main/java/net/bramp/sql/ResultSetFilter.java new file mode 100644 index 0000000..48ec247 --- /dev/null +++ b/src/main/java/net/bramp/sql/ResultSetFilter.java @@ -0,0 +1,876 @@ +package net.bramp.sql; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.net.URL; +import java.sql.Array; +import java.sql.Blob; +import java.sql.Clob; +import java.sql.Date; +import java.sql.NClob; +import java.sql.Ref; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.RowId; +import java.sql.SQLException; +import java.sql.SQLWarning; +import java.sql.SQLXML; +import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.Calendar; +import java.util.Map; + +import javax.annotation.Nonnull; + +/** + * Wraps an existing ResultSet, and filters out rows that don't match the predicate + * + * @author bramp + * + */ +public class ResultSetFilter implements ResultSet { + final ResultSet rs; + final Predicate predicate; + + public interface Predicate { + /** + * Return true if this row is acceptable + * Return false if it is not + * Do not call next() + * @param rs + * @return + * @throws SQLException + */ + public boolean apply(@Nonnull ResultSet rs) throws SQLException; + } + + public ResultSetFilter(@Nonnull ResultSet rs, @Nonnull Predicate predicate) { + this.rs = rs; + this.predicate = predicate; + } + + public T unwrap(Class iface) throws SQLException { + return rs.unwrap(iface); + } + + public boolean isWrapperFor(Class iface) throws SQLException { + return rs.isWrapperFor(iface); + } + + public boolean next() throws SQLException { + while(rs.next()) { + if (predicate.apply(rs)) + return true; + } + return false; + } + + public void close() throws SQLException { + rs.close(); + } + + public boolean wasNull() throws SQLException { + return rs.wasNull(); + } + + public String getString(int columnIndex) throws SQLException { + return rs.getString(columnIndex); + } + + public boolean getBoolean(int columnIndex) throws SQLException { + return rs.getBoolean(columnIndex); + } + + public byte getByte(int columnIndex) throws SQLException { + return rs.getByte(columnIndex); + } + + public short getShort(int columnIndex) throws SQLException { + return rs.getShort(columnIndex); + } + + public int getInt(int columnIndex) throws SQLException { + return rs.getInt(columnIndex); + } + + public long getLong(int columnIndex) throws SQLException { + return rs.getLong(columnIndex); + } + + public float getFloat(int columnIndex) throws SQLException { + return rs.getFloat(columnIndex); + } + + public double getDouble(int columnIndex) throws SQLException { + return rs.getDouble(columnIndex); + } + + @Deprecated + public BigDecimal getBigDecimal(int columnIndex, int scale) + throws SQLException { + return rs.getBigDecimal(columnIndex, scale); + } + + public byte[] getBytes(int columnIndex) throws SQLException { + return rs.getBytes(columnIndex); + } + + public Date getDate(int columnIndex) throws SQLException { + return rs.getDate(columnIndex); + } + + public Time getTime(int columnIndex) throws SQLException { + return rs.getTime(columnIndex); + } + + public Timestamp getTimestamp(int columnIndex) throws SQLException { + return rs.getTimestamp(columnIndex); + } + + public InputStream getAsciiStream(int columnIndex) throws SQLException { + return rs.getAsciiStream(columnIndex); + } + + @Deprecated + public InputStream getUnicodeStream(int columnIndex) throws SQLException { + return rs.getUnicodeStream(columnIndex); + } + + public InputStream getBinaryStream(int columnIndex) throws SQLException { + return rs.getBinaryStream(columnIndex); + } + + public String getString(String columnLabel) throws SQLException { + return rs.getString(columnLabel); + } + + public boolean getBoolean(String columnLabel) throws SQLException { + return rs.getBoolean(columnLabel); + } + + public byte getByte(String columnLabel) throws SQLException { + return rs.getByte(columnLabel); + } + + public short getShort(String columnLabel) throws SQLException { + return rs.getShort(columnLabel); + } + + public int getInt(String columnLabel) throws SQLException { + return rs.getInt(columnLabel); + } + + public long getLong(String columnLabel) throws SQLException { + return rs.getLong(columnLabel); + } + + public float getFloat(String columnLabel) throws SQLException { + return rs.getFloat(columnLabel); + } + + public double getDouble(String columnLabel) throws SQLException { + return rs.getDouble(columnLabel); + } + + @Deprecated + public BigDecimal getBigDecimal(String columnLabel, int scale) + throws SQLException { + return rs.getBigDecimal(columnLabel, scale); + } + + public byte[] getBytes(String columnLabel) throws SQLException { + return rs.getBytes(columnLabel); + } + + public Date getDate(String columnLabel) throws SQLException { + return rs.getDate(columnLabel); + } + + public Time getTime(String columnLabel) throws SQLException { + return rs.getTime(columnLabel); + } + + public Timestamp getTimestamp(String columnLabel) throws SQLException { + return rs.getTimestamp(columnLabel); + } + + public InputStream getAsciiStream(String columnLabel) throws SQLException { + return rs.getAsciiStream(columnLabel); + } + + @Deprecated + public InputStream getUnicodeStream(String columnLabel) throws SQLException { + return rs.getUnicodeStream(columnLabel); + } + + public InputStream getBinaryStream(String columnLabel) throws SQLException { + return rs.getBinaryStream(columnLabel); + } + + public SQLWarning getWarnings() throws SQLException { + return rs.getWarnings(); + } + + public void clearWarnings() throws SQLException { + rs.clearWarnings(); + } + + public String getCursorName() throws SQLException { + return rs.getCursorName(); + } + + public ResultSetMetaData getMetaData() throws SQLException { + return rs.getMetaData(); + } + + public Object getObject(int columnIndex) throws SQLException { + return rs.getObject(columnIndex); + } + + public Object getObject(String columnLabel) throws SQLException { + return rs.getObject(columnLabel); + } + + public int findColumn(String columnLabel) throws SQLException { + return rs.findColumn(columnLabel); + } + + public Reader getCharacterStream(int columnIndex) throws SQLException { + return rs.getCharacterStream(columnIndex); + } + + public Reader getCharacterStream(String columnLabel) throws SQLException { + return rs.getCharacterStream(columnLabel); + } + + public BigDecimal getBigDecimal(int columnIndex) throws SQLException { + return rs.getBigDecimal(columnIndex); + } + + public BigDecimal getBigDecimal(String columnLabel) throws SQLException { + return rs.getBigDecimal(columnLabel); + } + + public boolean isBeforeFirst() throws SQLException { + return rs.isBeforeFirst(); + } + + public boolean isAfterLast() throws SQLException { + return rs.isAfterLast(); + } + + public boolean isFirst() throws SQLException { + return rs.isFirst(); + } + + public boolean isLast() throws SQLException { + return rs.isLast(); + } + + public void beforeFirst() throws SQLException { + rs.beforeFirst(); + } + + public void afterLast() throws SQLException { + rs.afterLast(); + } + + public boolean first() throws SQLException { + return rs.first(); + } + + public boolean last() throws SQLException { + return rs.last(); + } + + public int getRow() throws SQLException { + return rs.getRow(); + } + + public boolean absolute(int row) throws SQLException { + return rs.absolute(row); + } + + public boolean relative(int rows) throws SQLException { + return rs.relative(rows); + } + + public boolean previous() throws SQLException { + return rs.previous(); + } + + public void setFetchDirection(int direction) throws SQLException { + rs.setFetchDirection(direction); + } + + public int getFetchDirection() throws SQLException { + return rs.getFetchDirection(); + } + + public void setFetchSize(int rows) throws SQLException { + rs.setFetchSize(rows); + } + + public int getFetchSize() throws SQLException { + return rs.getFetchSize(); + } + + public int getType() throws SQLException { + return rs.getType(); + } + + public int getConcurrency() throws SQLException { + return rs.getConcurrency(); + } + + public boolean rowUpdated() throws SQLException { + return rs.rowUpdated(); + } + + public boolean rowInserted() throws SQLException { + return rs.rowInserted(); + } + + public boolean rowDeleted() throws SQLException { + return rs.rowDeleted(); + } + + public void updateNull(int columnIndex) throws SQLException { + rs.updateNull(columnIndex); + } + + public void updateBoolean(int columnIndex, boolean x) throws SQLException { + rs.updateBoolean(columnIndex, x); + } + + public void updateByte(int columnIndex, byte x) throws SQLException { + rs.updateByte(columnIndex, x); + } + + public void updateShort(int columnIndex, short x) throws SQLException { + rs.updateShort(columnIndex, x); + } + + public void updateInt(int columnIndex, int x) throws SQLException { + rs.updateInt(columnIndex, x); + } + + public void updateLong(int columnIndex, long x) throws SQLException { + rs.updateLong(columnIndex, x); + } + + public void updateFloat(int columnIndex, float x) throws SQLException { + rs.updateFloat(columnIndex, x); + } + + public void updateDouble(int columnIndex, double x) throws SQLException { + rs.updateDouble(columnIndex, x); + } + + public void updateBigDecimal(int columnIndex, BigDecimal x) + throws SQLException { + rs.updateBigDecimal(columnIndex, x); + } + + public void updateString(int columnIndex, String x) throws SQLException { + rs.updateString(columnIndex, x); + } + + public void updateBytes(int columnIndex, byte[] x) throws SQLException { + rs.updateBytes(columnIndex, x); + } + + public void updateDate(int columnIndex, Date x) throws SQLException { + rs.updateDate(columnIndex, x); + } + + public void updateTime(int columnIndex, Time x) throws SQLException { + rs.updateTime(columnIndex, x); + } + + public void updateTimestamp(int columnIndex, Timestamp x) + throws SQLException { + rs.updateTimestamp(columnIndex, x); + } + + public void updateAsciiStream(int columnIndex, InputStream x, int length) + throws SQLException { + rs.updateAsciiStream(columnIndex, x, length); + } + + public void updateBinaryStream(int columnIndex, InputStream x, int length) + throws SQLException { + rs.updateBinaryStream(columnIndex, x, length); + } + + public void updateCharacterStream(int columnIndex, Reader x, int length) + throws SQLException { + rs.updateCharacterStream(columnIndex, x, length); + } + + public void updateObject(int columnIndex, Object x, int scaleOrLength) + throws SQLException { + rs.updateObject(columnIndex, x, scaleOrLength); + } + + public void updateObject(int columnIndex, Object x) throws SQLException { + rs.updateObject(columnIndex, x); + } + + public void updateNull(String columnLabel) throws SQLException { + rs.updateNull(columnLabel); + } + + public void updateBoolean(String columnLabel, boolean x) + throws SQLException { + rs.updateBoolean(columnLabel, x); + } + + public void updateByte(String columnLabel, byte x) throws SQLException { + rs.updateByte(columnLabel, x); + } + + public void updateShort(String columnLabel, short x) throws SQLException { + rs.updateShort(columnLabel, x); + } + + public void updateInt(String columnLabel, int x) throws SQLException { + rs.updateInt(columnLabel, x); + } + + public void updateLong(String columnLabel, long x) throws SQLException { + rs.updateLong(columnLabel, x); + } + + public void updateFloat(String columnLabel, float x) throws SQLException { + rs.updateFloat(columnLabel, x); + } + + public void updateDouble(String columnLabel, double x) throws SQLException { + rs.updateDouble(columnLabel, x); + } + + public void updateBigDecimal(String columnLabel, BigDecimal x) + throws SQLException { + rs.updateBigDecimal(columnLabel, x); + } + + public void updateString(String columnLabel, String x) throws SQLException { + rs.updateString(columnLabel, x); + } + + public void updateBytes(String columnLabel, byte[] x) throws SQLException { + rs.updateBytes(columnLabel, x); + } + + public void updateDate(String columnLabel, Date x) throws SQLException { + rs.updateDate(columnLabel, x); + } + + public void updateTime(String columnLabel, Time x) throws SQLException { + rs.updateTime(columnLabel, x); + } + + public void updateTimestamp(String columnLabel, Timestamp x) + throws SQLException { + rs.updateTimestamp(columnLabel, x); + } + + public void updateAsciiStream(String columnLabel, InputStream x, int length) + throws SQLException { + rs.updateAsciiStream(columnLabel, x, length); + } + + public void updateBinaryStream(String columnLabel, InputStream x, int length) + throws SQLException { + rs.updateBinaryStream(columnLabel, x, length); + } + + public void updateCharacterStream(String columnLabel, Reader reader, + int length) throws SQLException { + rs.updateCharacterStream(columnLabel, reader, length); + } + + public void updateObject(String columnLabel, Object x, int scaleOrLength) + throws SQLException { + rs.updateObject(columnLabel, x, scaleOrLength); + } + + public void updateObject(String columnLabel, Object x) throws SQLException { + rs.updateObject(columnLabel, x); + } + + public void insertRow() throws SQLException { + rs.insertRow(); + } + + public void updateRow() throws SQLException { + rs.updateRow(); + } + + public void deleteRow() throws SQLException { + rs.deleteRow(); + } + + public void refreshRow() throws SQLException { + rs.refreshRow(); + } + + public void cancelRowUpdates() throws SQLException { + rs.cancelRowUpdates(); + } + + public void moveToInsertRow() throws SQLException { + rs.moveToInsertRow(); + } + + public void moveToCurrentRow() throws SQLException { + rs.moveToCurrentRow(); + } + + public Statement getStatement() throws SQLException { + return rs.getStatement(); + } + + public Object getObject(int columnIndex, Map> map) + throws SQLException { + return rs.getObject(columnIndex, map); + } + + public Ref getRef(int columnIndex) throws SQLException { + return rs.getRef(columnIndex); + } + + public Blob getBlob(int columnIndex) throws SQLException { + return rs.getBlob(columnIndex); + } + + public Clob getClob(int columnIndex) throws SQLException { + return rs.getClob(columnIndex); + } + + public Array getArray(int columnIndex) throws SQLException { + return rs.getArray(columnIndex); + } + + public Object getObject(String columnLabel, Map> map) + throws SQLException { + return rs.getObject(columnLabel, map); + } + + public Ref getRef(String columnLabel) throws SQLException { + return rs.getRef(columnLabel); + } + + public Blob getBlob(String columnLabel) throws SQLException { + return rs.getBlob(columnLabel); + } + + public Clob getClob(String columnLabel) throws SQLException { + return rs.getClob(columnLabel); + } + + public Array getArray(String columnLabel) throws SQLException { + return rs.getArray(columnLabel); + } + + public Date getDate(int columnIndex, Calendar cal) throws SQLException { + return rs.getDate(columnIndex, cal); + } + + public Date getDate(String columnLabel, Calendar cal) throws SQLException { + return rs.getDate(columnLabel, cal); + } + + public Time getTime(int columnIndex, Calendar cal) throws SQLException { + return rs.getTime(columnIndex, cal); + } + + public Time getTime(String columnLabel, Calendar cal) throws SQLException { + return rs.getTime(columnLabel, cal); + } + + public Timestamp getTimestamp(int columnIndex, Calendar cal) + throws SQLException { + return rs.getTimestamp(columnIndex, cal); + } + + public Timestamp getTimestamp(String columnLabel, Calendar cal) + throws SQLException { + return rs.getTimestamp(columnLabel, cal); + } + + public URL getURL(int columnIndex) throws SQLException { + return rs.getURL(columnIndex); + } + + public URL getURL(String columnLabel) throws SQLException { + return rs.getURL(columnLabel); + } + + public void updateRef(int columnIndex, Ref x) throws SQLException { + rs.updateRef(columnIndex, x); + } + + public void updateRef(String columnLabel, Ref x) throws SQLException { + rs.updateRef(columnLabel, x); + } + + public void updateBlob(int columnIndex, Blob x) throws SQLException { + rs.updateBlob(columnIndex, x); + } + + public void updateBlob(String columnLabel, Blob x) throws SQLException { + rs.updateBlob(columnLabel, x); + } + + public void updateClob(int columnIndex, Clob x) throws SQLException { + rs.updateClob(columnIndex, x); + } + + public void updateClob(String columnLabel, Clob x) throws SQLException { + rs.updateClob(columnLabel, x); + } + + public void updateArray(int columnIndex, Array x) throws SQLException { + rs.updateArray(columnIndex, x); + } + + public void updateArray(String columnLabel, Array x) throws SQLException { + rs.updateArray(columnLabel, x); + } + + public RowId getRowId(int columnIndex) throws SQLException { + return rs.getRowId(columnIndex); + } + + public RowId getRowId(String columnLabel) throws SQLException { + return rs.getRowId(columnLabel); + } + + public void updateRowId(int columnIndex, RowId x) throws SQLException { + rs.updateRowId(columnIndex, x); + } + + public void updateRowId(String columnLabel, RowId x) throws SQLException { + rs.updateRowId(columnLabel, x); + } + + public int getHoldability() throws SQLException { + return rs.getHoldability(); + } + + public boolean isClosed() throws SQLException { + return rs.isClosed(); + } + + public void updateNString(int columnIndex, String nString) + throws SQLException { + rs.updateNString(columnIndex, nString); + } + + public void updateNString(String columnLabel, String nString) + throws SQLException { + rs.updateNString(columnLabel, nString); + } + + public void updateNClob(int columnIndex, NClob nClob) throws SQLException { + rs.updateNClob(columnIndex, nClob); + } + + public void updateNClob(String columnLabel, NClob nClob) + throws SQLException { + rs.updateNClob(columnLabel, nClob); + } + + public NClob getNClob(int columnIndex) throws SQLException { + return rs.getNClob(columnIndex); + } + + public NClob getNClob(String columnLabel) throws SQLException { + return rs.getNClob(columnLabel); + } + + public SQLXML getSQLXML(int columnIndex) throws SQLException { + return rs.getSQLXML(columnIndex); + } + + public SQLXML getSQLXML(String columnLabel) throws SQLException { + return rs.getSQLXML(columnLabel); + } + + public void updateSQLXML(int columnIndex, SQLXML xmlObject) + throws SQLException { + rs.updateSQLXML(columnIndex, xmlObject); + } + + public void updateSQLXML(String columnLabel, SQLXML xmlObject) + throws SQLException { + rs.updateSQLXML(columnLabel, xmlObject); + } + + public String getNString(int columnIndex) throws SQLException { + return rs.getNString(columnIndex); + } + + public String getNString(String columnLabel) throws SQLException { + return rs.getNString(columnLabel); + } + + public Reader getNCharacterStream(int columnIndex) throws SQLException { + return rs.getNCharacterStream(columnIndex); + } + + public Reader getNCharacterStream(String columnLabel) throws SQLException { + return rs.getNCharacterStream(columnLabel); + } + + public void updateNCharacterStream(int columnIndex, Reader x, long length) + throws SQLException { + rs.updateNCharacterStream(columnIndex, x, length); + } + + public void updateNCharacterStream(String columnLabel, Reader reader, + long length) throws SQLException { + rs.updateNCharacterStream(columnLabel, reader, length); + } + + public void updateAsciiStream(int columnIndex, InputStream x, long length) + throws SQLException { + rs.updateAsciiStream(columnIndex, x, length); + } + + public void updateBinaryStream(int columnIndex, InputStream x, long length) + throws SQLException { + rs.updateBinaryStream(columnIndex, x, length); + } + + public void updateCharacterStream(int columnIndex, Reader x, long length) + throws SQLException { + rs.updateCharacterStream(columnIndex, x, length); + } + + public void updateAsciiStream(String columnLabel, InputStream x, long length) + throws SQLException { + rs.updateAsciiStream(columnLabel, x, length); + } + + public void updateBinaryStream(String columnLabel, InputStream x, + long length) throws SQLException { + rs.updateBinaryStream(columnLabel, x, length); + } + + public void updateCharacterStream(String columnLabel, Reader reader, + long length) throws SQLException { + rs.updateCharacterStream(columnLabel, reader, length); + } + + public void updateBlob(int columnIndex, InputStream inputStream, long length) + throws SQLException { + rs.updateBlob(columnIndex, inputStream, length); + } + + public void updateBlob(String columnLabel, InputStream inputStream, + long length) throws SQLException { + rs.updateBlob(columnLabel, inputStream, length); + } + + public void updateClob(int columnIndex, Reader reader, long length) + throws SQLException { + rs.updateClob(columnIndex, reader, length); + } + + public void updateClob(String columnLabel, Reader reader, long length) + throws SQLException { + rs.updateClob(columnLabel, reader, length); + } + + public void updateNClob(int columnIndex, Reader reader, long length) + throws SQLException { + rs.updateNClob(columnIndex, reader, length); + } + + public void updateNClob(String columnLabel, Reader reader, long length) + throws SQLException { + rs.updateNClob(columnLabel, reader, length); + } + + public void updateNCharacterStream(int columnIndex, Reader x) + throws SQLException { + rs.updateNCharacterStream(columnIndex, x); + } + + public void updateNCharacterStream(String columnLabel, Reader reader) + throws SQLException { + rs.updateNCharacterStream(columnLabel, reader); + } + + public void updateAsciiStream(int columnIndex, InputStream x) + throws SQLException { + rs.updateAsciiStream(columnIndex, x); + } + + public void updateBinaryStream(int columnIndex, InputStream x) + throws SQLException { + rs.updateBinaryStream(columnIndex, x); + } + + public void updateCharacterStream(int columnIndex, Reader x) + throws SQLException { + rs.updateCharacterStream(columnIndex, x); + } + + public void updateAsciiStream(String columnLabel, InputStream x) + throws SQLException { + rs.updateAsciiStream(columnLabel, x); + } + + public void updateBinaryStream(String columnLabel, InputStream x) + throws SQLException { + rs.updateBinaryStream(columnLabel, x); + } + + public void updateCharacterStream(String columnLabel, Reader reader) + throws SQLException { + rs.updateCharacterStream(columnLabel, reader); + } + + public void updateBlob(int columnIndex, InputStream inputStream) + throws SQLException { + rs.updateBlob(columnIndex, inputStream); + } + + public void updateBlob(String columnLabel, InputStream inputStream) + throws SQLException { + rs.updateBlob(columnLabel, inputStream); + } + + public void updateClob(int columnIndex, Reader reader) throws SQLException { + rs.updateClob(columnIndex, reader); + } + + public void updateClob(String columnLabel, Reader reader) + throws SQLException { + rs.updateClob(columnLabel, reader); + } + + public void updateNClob(int columnIndex, Reader reader) throws SQLException { + rs.updateNClob(columnIndex, reader); + } + + public void updateNClob(String columnLabel, Reader reader) + throws SQLException { + rs.updateNClob(columnLabel, reader); + } + + public T getObject(int columnIndex, Class type) throws SQLException { + return rs.getObject(columnIndex, type); + } + + public T getObject(String columnLabel, Class type) + throws SQLException { + return rs.getObject(columnLabel, type); + } +} diff --git a/src/main/java/net/bramp/sql/ResultSets.java b/src/main/java/net/bramp/sql/ResultSets.java new file mode 100644 index 0000000..33faf2b --- /dev/null +++ b/src/main/java/net/bramp/sql/ResultSets.java @@ -0,0 +1,21 @@ +package net.bramp.sql; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public final class ResultSets { + private ResultSets() {} + + public static String toString(ResultSet rs) throws SQLException { + StringBuilder sb = new StringBuilder(); + int cols = rs.getMetaData().getColumnCount(); + for (int i = 1; i <= cols; i++) { + sb.append('"').append( rs.getString(i) ).append('"').append( ", "); + } + + if (cols > 0) + sb.setLength( sb.length() - 2); + + return sb.toString(); + } +} diff --git a/src/test/java/net/bramp/db_patterns/queues/MySQLBasedQueueTests.java b/src/test/java/net/bramp/db_patterns/queues/MySQLBasedQueueTests.java index 8d0f379..db792e8 100644 --- a/src/test/java/net/bramp/db_patterns/queues/MySQLBasedQueueTests.java +++ b/src/test/java/net/bramp/db_patterns/queues/MySQLBasedQueueTests.java @@ -34,7 +34,7 @@ public void setup() { @After public void cleanupDatabase() throws SQLException { queue.clear(); - queue.cleanupAll(); + queue.cleanupAll(10); assertEmpty(); } From ed8988f9199460daf3e28d86a1fcdcd9ab0429a3 Mon Sep 17 00:00:00 2001 From: Matzz Date: Tue, 15 Jul 2014 11:39:03 +0200 Subject: [PATCH 08/29] gitignore --- .gitignore | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.gitignore b/.gitignore index c5d92c3..5aa0adc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,30 @@ +*target* +*.jar +*.war +*.ear +*.class + +# eclipse specific git ignore +*.pydevproject +.project +.metadata +bin/** +tmp/** +tmp/**/* +*.tmp +*.bak +*.swp +*~.nib +local.properties +.classpath +.settings/ +.loadpath + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch target/ pom.xml.tag pom.xml.releaseBackup From 2b8042ee7da8c175f1ac599a1357e6d69107e6f3 Mon Sep 17 00:00:00 2001 From: Matzz Date: Wed, 16 Jul 2014 12:35:28 +0200 Subject: [PATCH 09/29] Delayed queue fixes + tests --- .../queues/AbstractBlockingQueue.java | 26 +++- .../queues/AbstractMySQLQueue.java | 30 +++- .../queues/MySQLBasedDelayQueue.java | 73 ++++++++- .../serializator/DefaultSerializator.java | 1 + .../queues/AbstractMySQLBasedQueueTest.java | 135 ++++++++++++++++ .../db_patterns/queues/DelayedString.java | 56 +++++++ .../queues/MySQLBasedDelayQueueTests.java | 145 ++++++------------ .../queues/MySQLBasedQueueTests.java | 100 ------------ 8 files changed, 353 insertions(+), 213 deletions(-) create mode 100644 src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java create mode 100644 src/test/java/net/bramp/db_patterns/queues/DelayedString.java delete mode 100644 src/test/java/net/bramp/db_patterns/queues/MySQLBasedQueueTests.java diff --git a/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java b/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java index 2c6d2f9..3527691 100644 --- a/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java @@ -12,14 +12,17 @@ */ abstract class AbstractBlockingQueue implements BlockingQueue { + @Override public boolean isEmpty() { return size() == 0; } + @Override public boolean offer(E e) { return add(e); } + @Override public E element() { E head = peek(); if (head == null) @@ -27,6 +30,7 @@ public E element() { return head; } + @Override public E remove() { E head = poll(); if (head == null) @@ -37,6 +41,7 @@ public E remove() { /** * Blocks */ + @Override public E take() throws InterruptedException { // We loop around trying to get a item, blocking at most a minute at // a time this allows us to be interrupted @@ -53,6 +58,7 @@ public E take() throws InterruptedException { /** * Blocks */ + @Override public void put(E e) throws InterruptedException { add(e); } @@ -60,6 +66,7 @@ public void put(E e) throws InterruptedException { /** * No blocking */ + @Override public int drainTo(Collection c) { return drainTo(c, Integer.MAX_VALUE); } @@ -67,6 +74,7 @@ public int drainTo(Collection c) { /** * No blocking */ + @Override public int drainTo(Collection c, int maxElements) { if (c == this) throw new IllegalArgumentException("Draining to self is not supported"); @@ -83,18 +91,13 @@ public int drainTo(Collection c, int maxElements) { return maxElements - count; } - - public void clear() { - // Lazy! just keep poll'ng them off - while (poll() != null) { - // Nothing - } - } + @Override public int remainingCapacity() { return Integer.MAX_VALUE; } + @Override public boolean offer(E e, long timeout, TimeUnit unit) throws InterruptedException { // Right now, we have no concept of a full queue, so we don't block on insert return offer(e); @@ -102,37 +105,46 @@ public boolean offer(E e, long timeout, TimeUnit unit) throws InterruptedExcepti ////// Nothing supported below + @Override public boolean contains(Object o) { throw new UnsupportedOperationException(); } + @Override public Iterator iterator() { throw new UnsupportedOperationException(); } + @Override public Object[] toArray() { throw new UnsupportedOperationException(); } + @Override public T[] toArray(T[] a) { throw new UnsupportedOperationException(); } + @Override public boolean remove(Object o) { throw new UnsupportedOperationException(); } + @Override public boolean containsAll(Collection c) { throw new UnsupportedOperationException(); } + @Override public boolean addAll(Collection c) { throw new UnsupportedOperationException(); } + @Override public boolean removeAll(Collection c) { throw new UnsupportedOperationException(); } + @Override public boolean retainAll(Collection c) { throw new UnsupportedOperationException(); } diff --git a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java index 56dfc6c..844970a 100644 --- a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java @@ -35,13 +35,16 @@ abstract class AbstractMySQLQueue extends AbstractBlockingQueue { protected String addQuery; protected String peekQuery; protected String[] pollQuery; + + protected String clearQuery = "DELETE FROM " + tableNamePlaceholder + + " WHERE queue_name = ? "; protected String cleanupQuery = "DELETE FROM " + tableNamePlaceholder - + " " + "WHERE acquired IS NOT NULL " + " AND queue_name = ? " + + " WHERE acquired IS NOT NULL " + " AND queue_name = ? " + " AND acquired < DATE_SUB(NOW(), INTERVAL ? DAY)"; protected String cleanupAllQuery = "DELETE FROM " + tableNamePlaceholder - + " " + "WHERE acquired IS NOT NULL " + + " WHERE acquired IS NOT NULL " + " AND acquired < DATE_SUB(NOW(), INTERVAL ? DAY)"; protected String sizeQuery = "SELECT COUNT(*) FROM queue WHERE acquired IS NULL AND queue_name = ?"; @@ -237,6 +240,25 @@ public E poll(long timeout, TimeUnit unit) throws InterruptedException { return head; } + + @Override + public void clear() { + Connection c; + try { + c = ds.getConnection(); + try { + CallableStatement s = c.prepareCall(getClearQuery()); + s.setString(1, queueName); + s.execute(); + + } finally { + c.close(); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + /** * Legacy cleanupAll * @@ -339,6 +361,10 @@ protected String[] getPollQuery() { protected String getSizeQuery() { return setTable(sizeQuery); } + + protected String getClearQuery() { + return setTable(clearQuery); + } protected String getCleanupQuery() { return setTable(cleanupQuery); diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java index c072391..ebbde7b 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java @@ -1,9 +1,17 @@ package net.bramp.db_patterns.queues; +import java.sql.Connection; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Timer; +import java.util.TimerTask; import java.util.concurrent.Delayed; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import javax.sql.DataSource; @@ -27,8 +35,8 @@ */ public class MySQLBasedDelayQueue extends AbstractMySQLQueue { - - protected String closestDelayQuery = "SELECT min(delayed_to)-NOW() FROM " + tableNamePlaceholder + + protected String closestDelayQuery = "SELECT min(TIME_TO_SEC(TIMEDIFF(delayed_to,NOW()))) FROM " + tableNamePlaceholder + " WHERE acquired IS NULL AND queue_name = ?"; protected String delayCondition = "AND (delayed_to<=NOW() OR delayed_to is null) "; @@ -81,13 +89,66 @@ protected void setAddParameters(E value, PreparedStatement s) setValueToStatment(s, 4, value); } + /** + * + * @return delay in seconds + * @throws SQLException + */ + protected long getClosestDelay() throws SQLException { + int minDelay = 0; + + String query = setTable(closestDelayQuery); + Connection c = ds.getConnection(); + + PreparedStatement s = c.prepareStatement(query); + s.setString(1, queueName); + if (s.execute()) { + ResultSet rs = s.getResultSet(); + if (rs != null && rs.next()) { + minDelay = rs.getInt(1); + } + } + return minDelay; + } + + protected ScheduledExecutorService wakeupScheduler = Executors.newScheduledThreadPool(1); + protected ScheduledFuture wakeupTask = null; + protected class WakeupTask implements Runnable { + @Override + public void run() { + condition.signal(); + synchronized(wakeupScheduler) { + wakeupScheduler.schedule(new Runnable() { + @Override + public void run() { + wakeupThread(); + } + }, 1, TimeUnit.SECONDS); + } + } + }; + @Override protected void wakeupThread() { - condition.signal(); - } + synchronized (wakeupScheduler) { + long delaySeconds; + try { + delaySeconds = getClosestDelay(); + if(delaySeconds<=0) { + delaySeconds = 1; + } + } catch (SQLException e) { + delaySeconds = 1; + e.printStackTrace(); + } + if(wakeupTask!=null && wakeupTask.getDelay(TimeUnit.SECONDS)>delaySeconds) { + wakeupTask.cancel(false); + } + if(wakeupTask==null || wakeupTask.isDone() || wakeupTask.isCancelled()) { + wakeupTask = wakeupScheduler.schedule(new WakeupTask(), delaySeconds, TimeUnit.SECONDS); + } - protected long getClosestDelay() { - return 1; + } } } diff --git a/src/main/java/net/bramp/serializator/DefaultSerializator.java b/src/main/java/net/bramp/serializator/DefaultSerializator.java index 8d109f9..4e98178 100644 --- a/src/main/java/net/bramp/serializator/DefaultSerializator.java +++ b/src/main/java/net/bramp/serializator/DefaultSerializator.java @@ -20,6 +20,7 @@ public byte[] serialize(E obj) { array = out.toByteArray(); } catch (IOException e) { objectOut.close(); + throw e; } } finally { out.close(); diff --git a/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java b/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java new file mode 100644 index 0000000..9bd9dc4 --- /dev/null +++ b/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java @@ -0,0 +1,135 @@ +package net.bramp.db_patterns.queues; + +import static org.junit.Assert.*; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collection; +import java.util.concurrent.TimeUnit; + +import javax.sql.DataSource; + +import net.bramp.db_patterns.DatabaseUtils; +import net.bramp.serializator.DefaultSerializator; + +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import com.google.common.base.Function; + +@RunWith(Parameterized.class) +public class AbstractMySQLBasedQueueTest { + + private Function valueFactory; + private AbstractMySQLQueue queue; + + final static long WAIT_FOR_TIMING_TEST = 300; // in ms + + @Parameterized.Parameters + public static Collection provideValueFactoriesAndQueues() { + String queueTableName = "queue"; + DataSource ds = DatabaseUtils.createDataSource(); + String me = "test"; + String queueName = java.util.UUID.randomUUID().toString(); + Function dsFactory = new Function() { + @Override + public DelayedString apply(String input) { + return new DelayedString(input, 0); + } + }; + Function sFactory = new Function() { + @Override + public String apply(String input) { + return input; + } + }; + + Object delayedQueue = new MySQLBasedDelayQueue(ds, + queueTableName, queueName, + new DefaultSerializator(), me); + Object standardQueue = new MySQLBasedQueue(ds, queueTableName, + queueName, String.class, me); + return Arrays.asList(new Object[][] { + new Object[] { dsFactory, delayedQueue }, + new Object[] { sFactory, standardQueue } + }); + } + + public AbstractMySQLBasedQueueTest(Function valueFactory, + AbstractMySQLQueue queue) { + this.valueFactory = valueFactory; + this.queue = queue; + } + + @After + public void cleanupDatabase() throws SQLException { + queue.clear(); + queue.cleanupAll(10); + assertEmpty(); + } + + protected void assertEmpty() { + assertTrue("Queue should start empty", queue.isEmpty()); + assertEquals("Queue should start empty", 0, queue.size()); + assertNull("Queue head should be null", queue.peek()); + } + + @Test + public void test() { + assertEmpty(); + + Object a = valueFactory.apply("A"); + assertTrue(queue.add(a)); + + assertEquals("Queue should contain one item", 1, queue.size()); + assertEquals("Queue head should be A", a, queue.peek()); + + Object b = valueFactory.apply("B"); + assertTrue(queue.add(b)); + + assertEquals("Queue should start empty", 2, queue.size()); + + assertEquals("Queue head should be A", a, queue.peek()); + assertEquals("Queue head should be A", a, queue.poll()); + + assertEquals("Queue should start empty", 1, queue.size()); + assertEquals("Queue head should be B", b, queue.peek()); + + assertEquals("Queue head should be B", b, queue.poll()); + + assertEmpty(); + } + + /* + * TODO We should change this to measure if take actually blocked forever + * + * @Test(timeout=5000) public void takeBlockingTest() throws + * InterruptedException { assertEmpty(); + * + * // This should block forever queue.take(); + * + * assertEmpty(); } + */ + + @Test(timeout = 1000) + public void pollBlockingTest() throws InterruptedException { + assertEmpty(); + + long wait = WAIT_FOR_TIMING_TEST; + + long now = System.currentTimeMillis(); + Object ret = queue.poll(wait, TimeUnit.MILLISECONDS); + long duration = System.currentTimeMillis() - now; + + assertNull("poll timed out", ret); + + assertTrue("We waited less than " + wait + "ms (actual:" + duration + + ")", duration >= wait); + assertTrue("We waited more than " + (wait * 1.2) + "ms (actual:" + + duration + ")", duration < wait * 1.2); + + assertEmpty(); + } +} diff --git a/src/test/java/net/bramp/db_patterns/queues/DelayedString.java b/src/test/java/net/bramp/db_patterns/queues/DelayedString.java new file mode 100644 index 0000000..b539e24 --- /dev/null +++ b/src/test/java/net/bramp/db_patterns/queues/DelayedString.java @@ -0,0 +1,56 @@ +package net.bramp.db_patterns.queues; + +import java.io.Serializable; +import java.util.concurrent.Delayed; +import java.util.concurrent.TimeUnit; + + +public class DelayedString implements Comparable, Delayed, Serializable { + + private static final long serialVersionUID = -574306132564575817L; + + private String str; + private long time; + private transient TimeUnit unit = TimeUnit.SECONDS; + + public DelayedString(String str, long seconds) { + this.str = str; + this.time = seconds + nowInSeconds(); + } + + public String get() { + return str; + } + + @Override + public int compareTo(Delayed o) { + Long l = o.getDelay(unit); + return l.compareTo(this.time); + } + + + @Override + public long getDelay(TimeUnit unit) { + return unit.convert(time - nowInSeconds(), unit); + } + + @Override + public boolean equals(Object o) { + if(o instanceof DelayedString) { + DelayedString v = (DelayedString) o; + return get().equals(v.get()); + } + else { + return false; + } + } + + @Override + public int hashCode() { + return get().hashCode(); + } + + private long nowInSeconds() { + return System.currentTimeMillis()/1000; + } +} diff --git a/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java b/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java index 7e3d534..ca2f58d 100644 --- a/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java +++ b/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java @@ -3,11 +3,7 @@ import static org.junit.Assert.*; import java.io.IOException; -import java.io.ObjectOutputStream; -import java.io.OutputStream; -import java.io.Serializable; import java.sql.SQLException; -import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; import javax.sql.DataSource; @@ -44,6 +40,7 @@ public void cleanupDatabase() throws SQLException { queue.clear(); queue.cleanupAll(0); assertEmpty(); + ds.getConnection().close(); } protected void assertEmpty() { @@ -52,127 +49,79 @@ protected void assertEmpty() { assertNull("Queue head should be null", queue.peek()); } + protected void assertBetween(long diff, long min, long max) { + assertTrue("Invalid range "+min+" <= "+diff+" <= "+max, min <= diff && diff <= max); + } + @Test - public void test() throws IOException { + public void getClosestDelayTest() throws SQLException { + long d, queueDelay; assertEmpty(); - - DelayedString a = new DelayedString("A", 0); - DelayedString b = new DelayedString("B", 0); - - assertTrue( queue.add(new DelayedString("A", 0)) ); - - assertEquals("Queue should contain one item", 1, queue.size()); - assertEquals("Queue head should be A", a, queue.peek()); - - assertTrue( queue.add(b) ); - - assertEquals("Queue should start empty", 2, queue.size()); - assertEquals("Queue head should be A", a, queue.peek()); - - assertEquals("Queue head should be A", a, queue.poll()); - - assertEquals("Queue should start empty", 1, queue.size()); - assertEquals("Queue head should be B", b, queue.peek()); + assertEquals(0, queue.getClosestDelay()); + d = 20; + queue.add(new DelayedString("A", d)); + queueDelay = queue.getClosestDelay(); + assertTrue("Closest delay should be close to added delay ["+d+"<="+(queueDelay+2)+"]", d <= queueDelay+2); + assertTrue("Closest delay should be less than last added ["+d+"<="+queueDelay+"]", queueDelay <= d); - assertEquals("Queue head should be B", b, queue.poll()); - - assertEmpty(); + queue.clear(); + d = 0; + queue.add(new DelayedString("A", d)); + assertTrue(queue.getClosestDelay()<=d); } + @Test public void nonBlockingPeekTest() throws IOException, InterruptedException { assertEmpty(); - long s = 2; - DelayedString a = new DelayedString("A", s); + long delayS = 2; + DelayedString a = new DelayedString("A", delayS); assertTrue( queue.add(a) ); assertEquals("Queue should contain one item", 1, queue.size()); assertNull("Queue head should be null", queue.peek()); - Thread.sleep(s*2*1000l); - assertEquals("Queue head should be null", a, queue.peek()); + Thread.sleep(delayS*2*1000l); + assertEquals("Queue head should not be expired task", a, queue.peek()); } - -// @Test(timeout=10000) -// public void delayedPollBlockingTest() throws IOException, InterruptedException, SQLException { -// assertEmpty(); -// long s = 2; -// DelayedString a = new DelayedString("A", s); -// -// assertTrue( queue.add(a) ); -// assertNull("Queue head should be null", queue.peek()); -// -// -// DelayedString ds = queue.poll(s*2, TimeUnit.SECONDS); -// -// assertEquals("Queue head should be object", a, ds); -// assertEmpty(); -// } - @Test(timeout=1000) - public void pollBlockingTest() throws InterruptedException { - assertEmpty(); - - long wait = WAIT_FOR_TIMING_TEST; - long now = System.currentTimeMillis(); - DelayedString ret = queue.poll(wait, TimeUnit.MILLISECONDS); - long duration = System.currentTimeMillis() - now; + @Test(timeout=10000) + public void delayedBlockingPollTest() throws IOException, InterruptedException, SQLException { + assertEmpty(); + long s = 2; + DelayedString a = new DelayedString("A", s); - assertNull("poll timed out", ret); + assertTrue( queue.add(a) ); + assertNull("Queue head should be null", queue.peek()); - assertTrue("We waited less than " + wait + "ms (actual:" + duration + ")", duration >= wait); - assertTrue("We waited more than " + (wait*1.2) + "ms (actual:" + duration + ")", duration < wait * 1.2); + DelayedString ds = queue.poll(s*2, TimeUnit.SECONDS); + assertEquals("Queue head should be object", a, ds); assertEmpty(); } - protected static class DelayedString implements Delayed, Serializable { - private static final long serialVersionUID = -574306132564575817L; - - private String str; - private long time; - private transient TimeUnit unit = TimeUnit.SECONDS; + @Test(timeout=10000) + public void multiplePoolTest() throws IOException, InterruptedException, SQLException { + assertEmpty(); + long delay = 2; - public DelayedString(String str, long seconds) { - this.str = str; - this.time = seconds + nowInSeconds(); - } + long tsStart = System.currentTimeMillis(); + char[] names = {'A', 'B', 'C', 'D'}; - public String get() { - return str; - } - - @Override - public int compareTo(Delayed o) { - Long l = o.getDelay(unit); - return l.compareTo(this.time); + int d = 0; + for(char name : names) { + assertTrue( queue.add(new DelayedString(String.valueOf(name), delay+d)) ); + d += 2; } - - @Override - public long getDelay(TimeUnit unit) { - return unit.convert(time - nowInSeconds(), unit); + for(char name : names) { + assertEquals("First str should be "+name, String.valueOf(name), queue.poll(delay+2, TimeUnit.SECONDS).get()); + assertBetween(System.currentTimeMillis() - tsStart, delay*1000-100, (delay+1)*1000+100); + tsStart = System.currentTimeMillis(); } - @Override - public boolean equals(Object o) { - if(o instanceof DelayedString) { - DelayedString v = (DelayedString) o; - return get().equals(v.get()); - } - else { - return false; - } - } - - @Override - public int hashCode() { - return get().hashCode(); - } - - private long nowInSeconds() { - return System.currentTimeMillis()/1000; - } + assertEmpty(); } + } diff --git a/src/test/java/net/bramp/db_patterns/queues/MySQLBasedQueueTests.java b/src/test/java/net/bramp/db_patterns/queues/MySQLBasedQueueTests.java deleted file mode 100644 index db792e8..0000000 --- a/src/test/java/net/bramp/db_patterns/queues/MySQLBasedQueueTests.java +++ /dev/null @@ -1,100 +0,0 @@ -package net.bramp.db_patterns.queues; - -import static org.junit.Assert.*; - -import java.sql.SQLException; -import java.util.concurrent.TimeUnit; - -import javax.sql.DataSource; - -import net.bramp.db_patterns.DatabaseUtils; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class MySQLBasedQueueTests { - - final static long WAIT_FOR_TIMING_TEST = 300; // in ms - - private String queueName; - private DataSource ds; - - private MySQLBasedQueue queue; - - @Before - public void setup() { - // Different queue name for each test (to avoid test clashes) - queueName = java.util.UUID.randomUUID().toString(); - ds = DatabaseUtils.createDataSource(); - - queue = new MySQLBasedQueue(ds, queueName, String.class, "test"); - } - - @After - public void cleanupDatabase() throws SQLException { - queue.clear(); - queue.cleanupAll(10); - assertEmpty(); - } - - protected void assertEmpty() { - assertTrue("Queue should start empty", queue.isEmpty()); - assertEquals("Queue should start empty", 0, queue.size()); - assertNull("Queue head should be null", queue.peek()); - } - - @Test - public void test() { - assertEmpty(); - - assertTrue( queue.add("A") ); - - assertEquals("Queue should contain one item", 1, queue.size()); - assertEquals("Queue head should be A", "A", queue.peek()); - - assertTrue( queue.add("B") ); - - assertEquals("Queue should start empty", 2, queue.size()); - assertEquals("Queue head should be A", "A", queue.peek()); - - assertEquals("Queue head should be A", "A", queue.poll()); - - assertEquals("Queue should start empty", 1, queue.size()); - assertEquals("Queue head should be B", "B", queue.peek()); - - assertEquals("Queue head should be B", "B", queue.poll()); - - assertEmpty(); - } - - /* TODO We should change this to measure if take actually blocked forever - @Test(timeout=5000) - public void takeBlockingTest() throws InterruptedException { - assertEmpty(); - - // This should block forever - queue.take(); - - assertEmpty(); - } - */ - - @Test(timeout=1000) - public void pollBlockingTest() throws InterruptedException { - assertEmpty(); - - long wait = WAIT_FOR_TIMING_TEST; - - long now = System.currentTimeMillis(); - String ret = queue.poll(wait, TimeUnit.MILLISECONDS); - long duration = System.currentTimeMillis() - now; - - assertNull("poll timed out", ret); - - assertTrue("We waited less than " + wait + "ms (actual:" + duration + ")", duration >= wait); - assertTrue("We waited more than " + (wait*1.2) + "ms (actual:" + duration + ")", duration < wait * 1.2); - - assertEmpty(); - } -} From d8f8afe37dd333bfd44e0896b8fd453a07640603 Mon Sep 17 00:00:00 2001 From: Matzz Date: Mon, 21 Jul 2014 14:26:47 +0200 Subject: [PATCH 10/29] Statusable queues --- .../queues/AbstractMySQLQueue.java | 289 +++++++++++++----- .../db_patterns/queues/CleanableQueue.java | 21 ++ .../queues/MySQLBasedDelayQueue.java | 41 ++- .../db_patterns/queues/MySQLBasedQueue.java | 42 ++- .../db_patterns/queues/StatusableQueue.java | 26 ++ .../queues/AbstractMySQLBasedQueueTest.java | 16 + .../queues/MySQLBasedDelayQueueTests.java | 1 + 7 files changed, 319 insertions(+), 117 deletions(-) create mode 100644 src/main/java/net/bramp/db_patterns/queues/CleanableQueue.java create mode 100644 src/main/java/net/bramp/db_patterns/queues/StatusableQueue.java diff --git a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java index 844970a..ad48dd7 100644 --- a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java @@ -20,8 +20,7 @@ * @param * @author bramp */ -abstract class AbstractMySQLQueue extends AbstractBlockingQueue { - +abstract class AbstractMySQLQueue extends AbstractBlockingQueue implements StatusableQueue, CleanableQueue { protected String me; protected DataSource ds; protected String queueName; @@ -33,6 +32,7 @@ abstract class AbstractMySQLQueue extends AbstractBlockingQueue { final static String tableNamePlaceholder = "%TABLE_NAME%"; protected String addQuery; + protected String peekQuery; protected String[] pollQuery; @@ -40,22 +40,31 @@ abstract class AbstractMySQLQueue extends AbstractBlockingQueue { + " WHERE queue_name = ? "; protected String cleanupQuery = "DELETE FROM " + tableNamePlaceholder - + " WHERE acquired IS NOT NULL " + " AND queue_name = ? " + + " WHERE acquired IS NOT NULL " + + " AND queue_name = ? " + " AND acquired < DATE_SUB(NOW(), INTERVAL ? DAY)"; protected String cleanupAllQuery = "DELETE FROM " + tableNamePlaceholder + " WHERE acquired IS NOT NULL " + " AND acquired < DATE_SUB(NOW(), INTERVAL ? DAY)"; + protected String updateStatusQuery = "UPDATE "+tableNamePlaceholder + +" SET status = ? " + + "WHERE id = ? " + + "LIMIT 1; "; + + protected String getStatusQuery = "SELECT status FROM queue WHERE id = ?"; + protected String sizeQuery = "SELECT COUNT(*) FROM queue WHERE acquired IS NULL AND queue_name = ?"; /** * Creates a new MySQL backed queue. Store values using statement setObject. * * @param ds + * @param queueTableName queue table name in database * @param queueTableName - * @param queueName - * @param type + * @param queueName queue name in database + * @param type value primitive type. Used to store value in database if serializator is not defined. * @param me * The name of this node, for storing in the database table */ @@ -69,11 +78,11 @@ public AbstractMySQLQueue(DataSource ds, String queueTableName, * Creates a new MySQL backed queue. Store values using serializator and * setBytes. * - * @param ds - * @param queueName - * @param serializator - * @param me - * The name of this node, for storing in the database table + * @param ds datasource + * @param queueTableName queue table name in database + * @param queueName queue name in database + * @param serializator used to store values + * @param me The name of this node, for storing in the database table */ public AbstractMySQLQueue(DataSource ds, String queueTableName, String queueName, Serializator serializator, String me) { @@ -90,6 +99,7 @@ protected AbstractMySQLQueue(DataSource ds, String tableName, this.me = me; } + @Override public boolean add(E value) { try { Connection c = ds.getConnection(); @@ -113,10 +123,8 @@ public boolean add(E value) { } } - /** - * No blocking - */ - public E peek() { + @Override + public ValueWithMetadata peekWithMetadata() { try { Connection c = ds.getConnection(); try { @@ -126,10 +134,12 @@ public E peek() { if (s.execute()) { ResultSet rs = s.getResultSet(); if (rs != null && rs.next()) { - return getValueFromResult(rs, 1); + return new ValueWithMetadata( + rs.getLong(1), + rs.getString(2), + getValueFromResult(rs, 3)); } } - return null; } finally { s.close(); @@ -144,10 +154,8 @@ public E peek() { } } - /** - * No blocking - */ - public E poll() { + @Override + public ValueWithMetadata pollWithMetadata() { try { Connection c = ds.getConnection(); String[] pollQuery = getPollQuery(); @@ -170,7 +178,10 @@ public E poll() { if (s3.execute()) { ResultSet rs = s3.getResultSet(); if (rs != null && rs.next()) { - return getValueFromResult(rs, 1); + return new ValueWithMetadata( + rs.getLong(1), + rs.getString(2), + getValueFromResult(rs, 3)); } } @@ -186,45 +197,19 @@ public E poll() { } } - public int size() { - try { - Connection c = ds.getConnection(); - try { - PreparedStatement s = c.prepareStatement(getSizeQuery()); - s.setString(1, queueName); - s.execute(); - - ResultSet rs = s.getResultSet(); - if (rs != null && rs.next()) - return rs.getInt(1); - - throw new RuntimeException("Failed to retreive size"); - - } finally { - c.close(); - } - - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - - /** - * Blocks until something is in the queue, up to timeout null if timeout - * occurs - */ - public E poll(long timeout, TimeUnit unit) throws InterruptedException { + @Override + public ValueWithMetadata pollWithMetadata(long timeout, TimeUnit unit) throws InterruptedException { final long deadlineMillis = System.currentTimeMillis() + unit.toMillis(timeout); final Date deadline = new Date(deadlineMillis); - E head = null; + ValueWithMetadata head = null; boolean stillWaiting = true; while (stillWaiting) { // Check if we can grab one - head = poll(); + head = pollWithMetadata(); if (head != null) break; @@ -241,6 +226,94 @@ public E poll(long timeout, TimeUnit unit) throws InterruptedException { } + @Override + public void updateStatus(long id, String newStatus) { + try { + Connection c = ds.getConnection(); + String updateStatusQuery = getUpdateStatusQuery(); + try { + CallableStatement s = c.prepareCall(updateStatusQuery); + s.setString(1, newStatus); + s.setLong(2, id); + s.execute(); + } finally { + c.close(); + } + + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + + @Override + public String getStatus(long id) { + try { + Connection c = ds.getConnection(); + String statusQuery = getStatusQuery(); + try { + CallableStatement s = c.prepareCall(statusQuery); + s.setLong(1, id); // Acquired by me + + if (s.execute()) { + ResultSet rs = s.getResultSet(); + if (rs != null && rs.next()) { + return rs.getString(1); + } + } + return null; + } finally { + c.close(); + } + + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public E peek() { + ValueWithMetadata item = peekWithMetadata(); + return item!=null ? item.value : null; + } + + @Override + public E poll() { + ValueWithMetadata item = pollWithMetadata(); + return item!=null ? item.value : null; + } + + @Override + public E poll(long timeout, TimeUnit unit) throws InterruptedException { + ValueWithMetadata item = pollWithMetadata(timeout, unit); + return item!=null ? item.value : null; + } + + @Override + public int size() { + try { + Connection c = ds.getConnection(); + try { + PreparedStatement s = c.prepareStatement(getSizeQuery()); + s.setString(1, queueName); + s.execute(); + + ResultSet rs = s.getResultSet(); + if (rs != null && rs.next()) + return rs.getInt(1); + + throw new RuntimeException("Failed to retreive size"); + + } finally { + c.close(); + } + + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override public void clear() { Connection c; @@ -269,6 +342,18 @@ public void cleanup() throws SQLException { cleanup(10); } + /** + * Legacy cleanupAll + * + * @deprecated + * @throws SQLException + */ + public void cleanupAll() throws SQLException { + cleanupAll(10); + } + + + @Override public void cleanup(int days) throws SQLException { Connection c = ds.getConnection(); try { @@ -282,21 +367,8 @@ public void cleanup(int days) throws SQLException { } } - /** - * Legacy cleanupAll - * - * @deprecated - * @throws SQLException - */ - public void cleanupAll() throws SQLException { - cleanupAll(10); - } - /** - * Cleans up all queues - * - * @throws SQLException - */ + @Override public void cleanupAll(int days) throws SQLException { Connection c = ds.getConnection(); try { @@ -309,29 +381,61 @@ public void cleanupAll(int days) throws SQLException { } } - protected void setAddParameters(E value, PreparedStatement s) throws SQLException { - } + /** + * Bind parameters to add query + * @param value to add + * @param statement + * @throws SQLException + */ + abstract protected void setAddParameters(E value, PreparedStatement statement) throws SQLException; + /** + * Binds table name to query + * @param query + * @return query with table name binded + */ protected String setTable(String query) { return query.replaceAll(tableNamePlaceholder, tableName); } + /** + * Escape table name to prevent SQL injection. + * @param tableName + * @return escaped against ` char + */ protected String escapeTableName(String tableName) { return "`" + tableName.replaceAll("`", "") + "`"; } + /** + * Wakes up one thread. + */ protected void wakeupThread() { condition.signal(); } + /** + * Get value from result set. Deserialize it if serializator defined otherwise getObject mehtod is used. + * @param rs + * @param index + * @return + * @throws SQLException + */ protected E getValueFromResult(ResultSet rs, int index) throws SQLException { if (serializator == null) { - return rs.getObject(1, type); + return rs.getObject(index, type); } else { return serializator.deserialize(rs.getBytes(index)); } } + /** + * Sets value to statement. If defined, serializator is used, otherwise setObject with type. + * @param s + * @param index + * @param obj + * @throws SQLException + */ protected void setValueToStatment(PreparedStatement s, int index, E obj) throws SQLException { if (serializator == null) { @@ -342,14 +446,26 @@ protected void setValueToStatment(PreparedStatement s, int index, E obj) } + /** + * Returns sql query for add operation with binded table name + * @return sql + */ protected String getAddQuery() { return setTable(addQuery); } - + + /** + * Returns sql query for add operation with binded table name + * @return sql + */ protected String getPeekQuery() { return setTable(peekQuery); } + /** + * Returns sql array for poll operation with binded table name + * @return sql array + */ protected String[] getPollQuery() { String[] queries = new String[pollQuery.length]; for(int i=0; i + * CREATE TABLE queue ( + * id INT UNSIGNED NOT NULL AUTO_INCREMENT, + * queue_name VARCHAR(255) NOT NULL, -- Queue name + * inserted TIMESTAMP NOT NULL, -- Time the row was inserted + * inserted_by VARCHAR(255) NOT NULL, -- and by who + * acquired TIMESTAMP NULL, -- Time the row was acquired + * acquired_by VARCHAR(255) NULL, -- and by who + * delayed_to TIMESTAMP NULL, -- Task delayed to + * value BLOB NOT NULL, -- The actual data + * status VARCHAR(255) NOT NULL DEFAULT 'NEW' -- The actual data + * PRIMARY KEY (id) + * ) ENGINE=INNODB DEFAULT CHARSET=UTF8; + *

* TODO Create efficient drainTo * * @param @@ -45,7 +47,7 @@ public class MySQLBasedDelayQueue extends + "(queue_name, inserted, inserted_by, delayed_to, value) values " + "(?, now(), ?, DATE_ADD(NOW(), INTERVAL ? SECOND), ?)"; - peekQuery = "SELECT value FROM "+tableNamePlaceholder+" WHERE " + peekQuery = "SELECT id, status, value FROM "+tableNamePlaceholder+" WHERE " + "acquired IS NULL " + delayCondition + "AND queue_name = ? " @@ -59,27 +61,24 @@ public class MySQLBasedDelayQueue extends + " acquired_by = ? " + "WHERE " + "acquired IS NULL " + delayCondition + "AND queue_name = ? " + "ORDER BY id ASC " + "LIMIT 1; ", - "SELECT value FROM "+tableNamePlaceholder+" WHERE id = @update_id" + "SELECT id, status, value FROM "+tableNamePlaceholder+" WHERE id = @update_id" }; } - /** - * {@inheritDoc} - */ + public MySQLBasedDelayQueue(DataSource ds, String queueTableName, String queueName, Class type, String me) { super(ds, queueTableName, queueName, type, me); } - /** - * {@inheritDoc} - */ + public MySQLBasedDelayQueue(DataSource ds, String queueTableName, String queueName, Serializator serializator, String me) { super(ds, queueTableName, queueName, serializator, me); } + @Override protected void setAddParameters(E value, PreparedStatement s) throws SQLException { @@ -90,7 +89,7 @@ protected void setAddParameters(E value, PreparedStatement s) } /** - * + * Returns closest task delay. * @return delay in seconds * @throws SQLException */ diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java index f0e1b36..f57b518 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java @@ -12,7 +12,7 @@ /** * A queue backed by MySQL - *

+ *

* CREATE TABLE queue ( * id INT UNSIGNED NOT NULL AUTO_INCREMENT, * queue_name VARCHAR(255) NOT NULL, -- Queue name @@ -21,6 +21,7 @@ * acquired TIMESTAMP NULL, -- Time the row was acquired * acquired_by VARCHAR(255) NULL, -- and by who * value BLOB NOT NULL, -- The actual data + * status VARCHAR(255) NOT NULL DEFAULT 'NEW' -- The actual data * PRIMARY KEY (id) * ) ENGINE=INNODB DEFAULT CHARSET=UTF8; *

@@ -30,26 +31,23 @@ * @author bramp */ public class MySQLBasedQueue extends AbstractMySQLQueue { - - protected Logger LOG = LoggerFactory.getLogger(MySQLBasedQueue.class); - { addQuery = "INSERT INTO "+tableNamePlaceholder+" " + "(queue_name, inserted, inserted_by, value) values " + "(?, now(), ?, ?)"; - peekQuery = "SELECT value FROM "+tableNamePlaceholder+" WHERE " + peekQuery = "SELECT id, status, value FROM "+tableNamePlaceholder+" WHERE " + "acquired IS NULL " + "AND queue_name = ? " + "ORDER BY id ASC LIMIT 1"; pollQuery = new String[] { "SET @update_id := -1; ", "UPDATE "+tableNamePlaceholder+" SET " - + " id = (SELECT @update_id := id), " - + " acquired = NOW(), " - + " acquired_by = ? " - + "WHERE " + "acquired IS NULL " - + "AND queue_name = ? " + "ORDER BY id ASC " + "LIMIT 1; ", - "SELECT value FROM "+tableNamePlaceholder+" WHERE id = @update_id" + + " id = (SELECT @update_id := id), " + + " acquired = NOW(), " + + " acquired_by = ? " + + "WHERE " + "acquired IS NULL " + + "AND queue_name = ? " + "ORDER BY id ASC " + "LIMIT 1; ", + "SELECT id, status, value FROM "+tableNamePlaceholder+" WHERE id = @update_id" }; } @@ -59,7 +57,7 @@ public class MySQLBasedQueue extends AbstractMySQLQueue { * @param ds * @param queueName * @param type - * @param me The name of this node, for storing in the database table + * @param me The name of this node, for storing in the database table * @deprecated */ public MySQLBasedQueue(DataSource ds, String queueName, Class type, String me) { @@ -67,24 +65,18 @@ public MySQLBasedQueue(DataSource ds, String queueName, Class type, String me this.type = type; } - @Override - protected void setAddParameters(E value, PreparedStatement s) throws SQLException { - s.setString(1, queueName); - s.setObject(2, me); // Inserted by me - setValueToStatment(s, 3, value); - } - - /** - * {@inheritDoc} - */ public MySQLBasedQueue(DataSource ds, String queueTableName, String queueName, Class type, String me) { super(ds, queueTableName, queueName, type, me); } - /** - * {@inheritDoc} - */ public MySQLBasedQueue(DataSource ds, String queueTableName, String queueName, Serializator serializator, String me) { super(ds, queueTableName, queueName, serializator, me); } + + @Override + protected void setAddParameters(E value, PreparedStatement s) throws SQLException { + s.setString(1, queueName); + s.setObject(2, me); // Inserted by me + setValueToStatment(s, 3, value); + } } diff --git a/src/main/java/net/bramp/db_patterns/queues/StatusableQueue.java b/src/main/java/net/bramp/db_patterns/queues/StatusableQueue.java new file mode 100644 index 0000000..2bfa08e --- /dev/null +++ b/src/main/java/net/bramp/db_patterns/queues/StatusableQueue.java @@ -0,0 +1,26 @@ +package net.bramp.db_patterns.queues; + +import java.util.concurrent.TimeUnit; + +public interface StatusableQueue { + public ValueWithMetadata pollWithMetadata(); + public ValueWithMetadata pollWithMetadata(long timeout, TimeUnit unit) throws InterruptedException; + public ValueWithMetadata peekWithMetadata(); + public void updateStatus(long id, String newStatus); + public String getStatus(long id); + + public static class ValueWithMetadata { + public final long id; + public final String status; + public final E value; + ValueWithMetadata(long id, String status, E value) { + this.id = id; + this.status = status; + this.value = value; + } + @Override + public String toString() { + return "ValueWithMetadata["+id+" "+status+" "+value+"]"; + } + } +} diff --git a/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java b/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java index 9bd9dc4..13c2157 100644 --- a/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java +++ b/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java @@ -3,13 +3,16 @@ import static org.junit.Assert.*; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.concurrent.TimeUnit; import javax.sql.DataSource; import net.bramp.db_patterns.DatabaseUtils; +import net.bramp.db_patterns.queues.StatusableQueue.ValueWithMetadata; import net.bramp.serializator.DefaultSerializator; import org.junit.After; @@ -102,6 +105,19 @@ public void test() { assertEmpty(); } + @Test + public void statusTest() { + assertEmpty(); + + Object a = valueFactory.apply("A"); + assertTrue(queue.add(a)); + ValueWithMetadata v = queue.peekWithMetadata(); + queue.updateStatus(v.id, "Test1"); + assertEquals(queue.getStatus(v.id), "Test1"); + queue.updateStatus(v.id, "Test2"); + assertEquals(queue.getStatus(v.id), "Test2"); + } + /* * TODO We should change this to measure if take actually blocked forever * diff --git a/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java b/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java index ca2f58d..22e50c0 100644 --- a/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java +++ b/src/test/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueueTests.java @@ -74,6 +74,7 @@ public void getClosestDelayTest() throws SQLException { @Test public void nonBlockingPeekTest() throws IOException, InterruptedException { + assertEmpty(); long delayS = 2; DelayedString a = new DelayedString("A", delayS); From b53c218aa9d5f755fb7f2fd136395460805e1cbd Mon Sep 17 00:00:00 2001 From: Mateusz Zakarczemny Date: Mon, 21 Jul 2014 14:53:33 +0200 Subject: [PATCH 11/29] Update README.md --- README.md | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6dfba2d..00114c3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ MySQL DB Patterns ================= -by [Andrew Brampton](http://bramp.net) 2013 +by [Andrew Brampton](http://bramp.net) 2013, contributed by [Mateusz Zakarczemny](https://github.com/Matzz) Intro ----- @@ -36,14 +36,16 @@ The MySQLSleepBasedCondition is based on the MySQL ``SLEEP()`` and ``KILL QUERY` The thread that is woken up is guaranteed to be the one that has waited the longest. -Queue +Blocking queue ----- A distributed MySQL backed Java BlockingQueue ```java DataSource ds = ... - BlockingQueue queue = new MySQLBasedQueue(ds, "queue name", String.class); + + //datasoruce, queue table, queuename, value type, thread name + BlockingQueue queue = new MySQLBasedQueue(ds, "queue", "queue name", String.class, "Worker1"); queue.add("Some String"); // on another thread (or process, or machine) @@ -55,6 +57,31 @@ A distributed MySQL backed Java BlockingQueue The MySQLBasedQueue uses the MySQLSleepBasedCondition to help form a blocking queue, that can work without polling the database for new work. +More complex types could be stored using serializator: +```java + Serializator serializator = new DefaultSerializator(); + BlockingQueue queue = new MySQLBasedQueue(ds, "queue", "queue name", serializator, "Worker1"); + MyType value = new MyType(...); + queue.add(value); +``` +DefaultSerializator serializes values using java ObjectOutputStream but other implementation might be passed to queue (eg. some custom JsonSerializer). + +DelayQueue +----------------- +A distributed MySQL backed Java DelayQueue + +```java + Serializator serializator = new DefaultSerializator(); // MyType must extends Delayed interface + DelayQueue queue = new MySQLBasedDelayQueue(ds, "queue", "queue name", serializator, "Worker1"); + MyDelayedType value = new MyDelayedType(10, TimeUnit.SECONDS); + queue.add(value); + queue.peek(); // equals null + Thread.sleep(11*1000); + queue.peek(); // equals value + +``` + + Build and Release ----------------- From cd0bc265c9a2fdfcde69b6b374361cd60fc522c9 Mon Sep 17 00:00:00 2001 From: Mateusz Zakarczemny Date: Mon, 21 Jul 2014 15:21:15 +0200 Subject: [PATCH 12/29] Statuses doc --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 00114c3..936638b 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,19 @@ A distributed MySQL backed Java DelayQueue ``` +Statuses +----------------- +MySQLBasedQueue, MySQLBasedDelayQueue implements StatusableQueue interface which enable seting queue item statuses. Statuses do not affect polling of items. They might be set at any time and to any value. They just provide convenient way of tracking item state. + +StatusableQueue brings such methods: +```java + public ValueWithMetadata pollWithMetadata(); + public ValueWithMetadata pollWithMetadata(long timeout, TimeUnit unit) throws InterruptedException; + public ValueWithMetadata peekWithMetadata(); + public void updateStatus(long id, String newStatus); + public String getStatus(long id); +``` +ValueWithMetadata class contains item id in queue, status and item value. Build and Release From 67a8a76542b6fd12ef08c2e286c208b18a2c03b6 Mon Sep 17 00:00:00 2001 From: Matzz Date: Thu, 24 Jul 2014 10:46:41 +0200 Subject: [PATCH 13/29] take with parameters --- .../queues/AbstractBlockingQueue.java | 16 -------------- .../queues/AbstractMySQLQueue.java | 21 +++++++++++++++++++ .../db_patterns/queues/StatusableQueue.java | 1 + .../queues/AbstractMySQLBasedQueueTest.java | 2 -- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java b/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java index 3527691..8c6956f 100644 --- a/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/AbstractBlockingQueue.java @@ -38,22 +38,6 @@ public E remove() { return head; } - /** - * Blocks - */ - @Override - public E take() throws InterruptedException { - // We loop around trying to get a item, blocking at most a minute at - // a time this allows us to be interrupted - E head = null; - while (head == null) { - if (Thread.interrupted()) - throw new InterruptedException(); - - head = poll(1, TimeUnit.MINUTES); - } - return head; - } /** * Blocks diff --git a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java index ad48dd7..7ece3c1 100644 --- a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java @@ -197,6 +197,21 @@ public ValueWithMetadata pollWithMetadata() { } } + + @Override + public ValueWithMetadata takeWithMetadata() throws InterruptedException { + // We loop around trying to get a item, blocking at most a minute at + // a time this allows us to be interrupted + ValueWithMetadata head = null; + while (head == null) { + if (Thread.interrupted()) + throw new InterruptedException(); + + head = pollWithMetadata(1, TimeUnit.MINUTES); + } + return head; + } + @Override public ValueWithMetadata pollWithMetadata(long timeout, TimeUnit unit) throws InterruptedException { @@ -282,6 +297,12 @@ public E poll() { ValueWithMetadata item = pollWithMetadata(); return item!=null ? item.value : null; } + + @Override + public E take() throws InterruptedException { + ValueWithMetadata item = takeWithMetadata(); + return item!=null ? item.value : null; + } @Override public E poll(long timeout, TimeUnit unit) throws InterruptedException { diff --git a/src/main/java/net/bramp/db_patterns/queues/StatusableQueue.java b/src/main/java/net/bramp/db_patterns/queues/StatusableQueue.java index 2bfa08e..b00f140 100644 --- a/src/main/java/net/bramp/db_patterns/queues/StatusableQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/StatusableQueue.java @@ -6,6 +6,7 @@ public interface StatusableQueue { public ValueWithMetadata pollWithMetadata(); public ValueWithMetadata pollWithMetadata(long timeout, TimeUnit unit) throws InterruptedException; public ValueWithMetadata peekWithMetadata(); + public ValueWithMetadata takeWithMetadata() throws InterruptedException; public void updateStatus(long id, String newStatus); public String getStatus(long id); diff --git a/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java b/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java index 13c2157..d70b4a8 100644 --- a/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java +++ b/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java @@ -3,10 +3,8 @@ import static org.junit.Assert.*; import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.List; import java.util.concurrent.TimeUnit; import javax.sql.DataSource; From fa1cd96770708a36789ba52d9344d34e948c0dde Mon Sep 17 00:00:00 2001 From: Matzz Date: Tue, 29 Jul 2014 13:56:33 +0200 Subject: [PATCH 14/29] code formatting --- .../java/net/bramp/db_patterns/queues/MySQLBasedQueue.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java index f57b518..534427d 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java @@ -45,8 +45,11 @@ public class MySQLBasedQueue extends AbstractMySQLQueue { + " id = (SELECT @update_id := id), " + " acquired = NOW(), " + " acquired_by = ? " - + "WHERE " + "acquired IS NULL " - + "AND queue_name = ? " + "ORDER BY id ASC " + "LIMIT 1; ", + + "WHERE " + + "acquired IS NULL " + + "AND queue_name = ? " + + "ORDER BY id ASC " + + "LIMIT 1; ", "SELECT id, status, value FROM "+tableNamePlaceholder+" WHERE id = @update_id" }; } From bbe62d09ec476766393c288237a3ad483d68104e Mon Sep 17 00:00:00 2001 From: Matzz Date: Tue, 29 Jul 2014 13:59:30 +0200 Subject: [PATCH 15/29] standalone version --- pom.xml | 109 ++++++++++++++++++++++++-------------------------------- 1 file changed, 46 insertions(+), 63 deletions(-) diff --git a/pom.xml b/pom.xml index efbdd12..3785d9b 100644 --- a/pom.xml +++ b/pom.xml @@ -1,9 +1,9 @@ 4.0.0 - net.bramp.db-patterns + matzz db-patterns - 0.2-SNAPSHOT + 0.3-SNAPSHOT jar DB Patterns @@ -25,12 +25,6 @@ - - https://github.com/bramp/db-patterns - scm:git:git@github.com:bramp/db-patterns.git - HEAD - - UTF-8 UTF-8 @@ -91,17 +85,6 @@ - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - @@ -156,50 +139,50 @@ - - org.apache.maven.plugins - maven-deploy-plugin - 2.8.1 - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.5 - - - sign-artifacts - verify - - sign - - - - - - - org.apache.maven.plugins - maven-release-plugin - 2.5 - - true - false - release - deploy nexus-staging:release - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.1 - true - - ossrh - https://oss.sonatype.org/ - true - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From d137f2845c63714954bc7d9dec99dee181d2a8f8 Mon Sep 17 00:00:00 2001 From: Matzz Date: Tue, 29 Jul 2014 14:21:28 +0200 Subject: [PATCH 16/29] v3.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3785d9b..92cd9e2 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ matzz db-patterns - 0.3-SNAPSHOT + 0.3.1 jar DB Patterns From 4ffc2b561ca5013bda57963ad53ba9228f1ed909 Mon Sep 17 00:00:00 2001 From: Matzz Date: Tue, 29 Jul 2014 16:13:26 +0200 Subject: [PATCH 17/29] mysql-connector dep --- pom.xml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 92cd9e2..6df4230 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ matzz db-patterns - 0.3.1 + 0.3.2 jar DB Patterns @@ -54,8 +54,7 @@ mysql mysql-connector-java - 5.1.27 - provided + 5.1.31 From 79ebf9c5e0d354428194d003d89291bc2a5b54e2 Mon Sep 17 00:00:00 2001 From: Matzz Date: Wed, 30 Jul 2014 12:51:21 +0200 Subject: [PATCH 18/29] configurable take blocking time --- .../queues/AbstractMySQLQueue.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java index 7ece3c1..71ca3ee 100644 --- a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java @@ -29,6 +29,11 @@ abstract class AbstractMySQLQueue extends AbstractBlockingQueue implements protected Class type = null; protected Serializator serializator = null; protected Condition condition; + /** + * time in seconds + */ + private volatile int takeBlockingTime = 60; + final static String tableNamePlaceholder = "%TABLE_NAME%"; protected String addQuery; @@ -73,6 +78,7 @@ public AbstractMySQLQueue(DataSource ds, String queueTableName, this(ds, queueTableName, queueName, me); this.type = type; } + /** * Creates a new MySQL backed queue. Store values using serializator and @@ -99,6 +105,23 @@ protected AbstractMySQLQueue(DataSource ds, String tableName, this.me = me; } + /** + * Gets take operation blocking time. Default 60s. + * Unit - seconds. + */ + public int getTakeBlockingTime() { + return takeBlockingTime; + } + + /** + * Sets take operation blocking time. Decrease it to enable faster interruption. + * Unit - seconds. + */ + public void setTakeBlockingTime(int takeBlockingTime) { + this.takeBlockingTime = takeBlockingTime; + } + + @Override public boolean add(E value) { try { @@ -207,7 +230,7 @@ public ValueWithMetadata takeWithMetadata() throws InterruptedException { if (Thread.interrupted()) throw new InterruptedException(); - head = pollWithMetadata(1, TimeUnit.MINUTES); + head = pollWithMetadata(takeBlockingTime, TimeUnit.SECONDS); } return head; } From 545c2d41be64d69920298a72c4d37cd0516321c6 Mon Sep 17 00:00:00 2001 From: Matzz Date: Wed, 30 Jul 2014 12:56:12 +0200 Subject: [PATCH 19/29] 0.3.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6df4230..f79ac31 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ matzz db-patterns - 0.3.2 + 0.3.3 jar DB Patterns From c64952a933acb8181013908f0103cc929cf14c69 Mon Sep 17 00:00:00 2001 From: Matzz Date: Tue, 12 Aug 2014 11:43:15 +0200 Subject: [PATCH 20/29] Added priorities --- README.md | 24 +++- pom.xml | 2 +- .../queues/AbstractMySQLQueue.java | 112 ++++++++++-------- .../queues/MySQLBasedDelayQueue.java | 47 ++++---- .../db_patterns/queues/MySQLBasedQueue.java | 44 +++---- .../db_patterns/queues/StatusableQueue.java | 27 ----- .../db_patterns/queues/ValueContainer.java | 44 +++++++ .../{ => interfaces}/CleanableQueue.java | 2 +- .../queues/interfaces/PriorityQueue.java | 5 + .../queues/interfaces/StatusableQueue.java | 12 ++ .../queues/interfaces/ValueWithMetadata.java | 7 ++ .../queues/interfaces/ValueWithPriority.java | 5 + .../queues/AbstractMySQLBasedQueueTest.java | 50 +++++++- .../db_patterns/queues/DelayedString.java | 5 + .../MultithreadMySQLBasedQueueTests.java | 6 +- .../queues/StressMySQLBasedQueueTests.java | 6 +- 16 files changed, 259 insertions(+), 139 deletions(-) delete mode 100644 src/main/java/net/bramp/db_patterns/queues/StatusableQueue.java create mode 100644 src/main/java/net/bramp/db_patterns/queues/ValueContainer.java rename src/main/java/net/bramp/db_patterns/queues/{ => interfaces}/CleanableQueue.java (90%) create mode 100644 src/main/java/net/bramp/db_patterns/queues/interfaces/PriorityQueue.java create mode 100644 src/main/java/net/bramp/db_patterns/queues/interfaces/StatusableQueue.java create mode 100644 src/main/java/net/bramp/db_patterns/queues/interfaces/ValueWithMetadata.java create mode 100644 src/main/java/net/bramp/db_patterns/queues/interfaces/ValueWithPriority.java diff --git a/README.md b/README.md index 936638b..c092b42 100644 --- a/README.md +++ b/README.md @@ -83,17 +83,29 @@ A distributed MySQL backed Java DelayQueue Statuses ----------------- -MySQLBasedQueue, MySQLBasedDelayQueue implements StatusableQueue interface which enable seting queue item statuses. Statuses do not affect polling of items. They might be set at any time and to any value. They just provide convenient way of tracking item state. +MySQLBasedQueue, MySQLBasedDelayQueue implements StatusableQueue interface which enables setting queue item statuses. Statuses do not affect polling of items. They might be set at any time and to any value. They just provides convenient way of tracking item state. -StatusableQueue brings such methods: +PriorityQueue brings such methods: ```java - public ValueWithMetadata pollWithMetadata(); - public ValueWithMetadata pollWithMetadata(long timeout, TimeUnit unit) throws InterruptedException; - public ValueWithMetadata peekWithMetadata(); + public V pollWithMetadata(); + public V pollWithMetadata(long timeout, TimeUnit unit) throws InterruptedException; + public V peekWithMetadata(); public void updateStatus(long id, String newStatus); public String getStatus(long id); ``` -ValueWithMetadata class contains item id in queue, status and item value. +Where V is class implementing ValueWithMetadata interface. ValueWithMetadata contains item id in queue, status and item value. + + +Priority +----------------- +MySQLBasedQueue, MySQLBasedDelayQueue implements PriorityQueue interface which enables setting items priority. The higher priority is, the earlier item will be polled from queue. + + +StatusableQueue brings such methods: +```java + public boolean add(E value, int priority); +``` +Default priority for add is 0. Priority could be retrieved from value metadata (see above) using getPriority method. Build and Release diff --git a/pom.xml b/pom.xml index f79ac31..526e901 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ matzz db-patterns - 0.3.3 + 0.3.4 jar DB Patterns diff --git a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java index 71ca3ee..4700774 100644 --- a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java @@ -12,6 +12,9 @@ import javax.sql.DataSource; import net.bramp.db_patterns.locks.MySQLSleepBasedCondition; +import net.bramp.db_patterns.queues.interfaces.CleanableQueue; +import net.bramp.db_patterns.queues.interfaces.PriorityQueue; +import net.bramp.db_patterns.queues.interfaces.StatusableQueue; import net.bramp.serializator.Serializator; /** @@ -20,7 +23,8 @@ * @param * @author bramp */ -abstract class AbstractMySQLQueue extends AbstractBlockingQueue implements StatusableQueue, CleanableQueue { +abstract class AbstractMySQLQueue extends AbstractBlockingQueue implements + StatusableQueue>, PriorityQueue, CleanableQueue { protected String me; protected DataSource ds; protected String queueName; @@ -33,7 +37,6 @@ abstract class AbstractMySQLQueue extends AbstractBlockingQueue implements * time in seconds */ private volatile int takeBlockingTime = 60; - final static String tableNamePlaceholder = "%TABLE_NAME%"; protected String addQuery; @@ -43,20 +46,17 @@ abstract class AbstractMySQLQueue extends AbstractBlockingQueue implements protected String clearQuery = "DELETE FROM " + tableNamePlaceholder + " WHERE queue_name = ? "; - + protected String cleanupQuery = "DELETE FROM " + tableNamePlaceholder - + " WHERE acquired IS NOT NULL " - + " AND queue_name = ? " + + " WHERE acquired IS NOT NULL " + " AND queue_name = ? " + " AND acquired < DATE_SUB(NOW(), INTERVAL ? DAY)"; protected String cleanupAllQuery = "DELETE FROM " + tableNamePlaceholder + " WHERE acquired IS NOT NULL " + " AND acquired < DATE_SUB(NOW(), INTERVAL ? DAY)"; - protected String updateStatusQuery = "UPDATE "+tableNamePlaceholder - +" SET status = ? " - + "WHERE id = ? " - + "LIMIT 1; "; + protected String updateStatusQuery = "UPDATE " + tableNamePlaceholder + + " SET status = ? " + "WHERE id = ? " + "LIMIT 1; "; protected String getStatusQuery = "SELECT status FROM queue WHERE id = ?"; @@ -78,7 +78,6 @@ public AbstractMySQLQueue(DataSource ds, String queueTableName, this(ds, queueTableName, queueName, me); this.type = type; } - /** * Creates a new MySQL backed queue. Store values using serializator and @@ -121,15 +120,19 @@ public void setTakeBlockingTime(int takeBlockingTime) { this.takeBlockingTime = takeBlockingTime; } - @Override public boolean add(E value) { + return add(value, ValueContainer.DEFAULT_PRIORRITY); + } + + @Override + public boolean add(E value, int priority) { try { Connection c = ds.getConnection(); try { PreparedStatement s = c.prepareStatement(getAddQuery()); try { - setAddParameters(value, s); + setAddParameters(value, priority, s); s.execute(); wakeupThread(); return true; @@ -145,9 +148,9 @@ public boolean add(E value) { throw new RuntimeException(e); } } - + @Override - public ValueWithMetadata peekWithMetadata() { + public ValueContainer peekWithMetadata() { try { Connection c = ds.getConnection(); try { @@ -157,10 +160,7 @@ public ValueWithMetadata peekWithMetadata() { if (s.execute()) { ResultSet rs = s.getResultSet(); if (rs != null && rs.next()) { - return new ValueWithMetadata( - rs.getLong(1), - rs.getString(2), - getValueFromResult(rs, 3)); + return valueContainerFromResult(rs); } } return null; @@ -178,7 +178,7 @@ public ValueWithMetadata peekWithMetadata() { } @Override - public ValueWithMetadata pollWithMetadata() { + public ValueContainer pollWithMetadata() { try { Connection c = ds.getConnection(); String[] pollQuery = getPollQuery(); @@ -201,10 +201,7 @@ public ValueWithMetadata pollWithMetadata() { if (s3.execute()) { ResultSet rs = s3.getResultSet(); if (rs != null && rs.next()) { - return new ValueWithMetadata( - rs.getLong(1), - rs.getString(2), - getValueFromResult(rs, 3)); + return valueContainerFromResult(rs); } } @@ -220,12 +217,11 @@ public ValueWithMetadata pollWithMetadata() { } } - @Override - public ValueWithMetadata takeWithMetadata() throws InterruptedException { + public ValueContainer takeWithMetadata() throws InterruptedException { // We loop around trying to get a item, blocking at most a minute at // a time this allows us to be interrupted - ValueWithMetadata head = null; + ValueContainer head = null; while (head == null) { if (Thread.interrupted()) throw new InterruptedException(); @@ -236,13 +232,14 @@ public ValueWithMetadata takeWithMetadata() throws InterruptedException { } @Override - public ValueWithMetadata pollWithMetadata(long timeout, TimeUnit unit) throws InterruptedException { + public ValueContainer pollWithMetadata(long timeout, TimeUnit unit) + throws InterruptedException { final long deadlineMillis = System.currentTimeMillis() + unit.toMillis(timeout); final Date deadline = new Date(deadlineMillis); - ValueWithMetadata head = null; + ValueContainer head = null; boolean stillWaiting = true; while (stillWaiting) { @@ -263,7 +260,6 @@ public ValueWithMetadata pollWithMetadata(long timeout, TimeUnit unit) throws return head; } - @Override public void updateStatus(long id, String newStatus) { try { @@ -283,7 +279,6 @@ public void updateStatus(long id, String newStatus) { } } - @Override public String getStatus(long id) { try { @@ -311,26 +306,26 @@ public String getStatus(long id) { @Override public E peek() { - ValueWithMetadata item = peekWithMetadata(); - return item!=null ? item.value : null; + ValueContainer item = peekWithMetadata(); + return item != null ? item.value : null; } @Override public E poll() { - ValueWithMetadata item = pollWithMetadata(); - return item!=null ? item.value : null; + ValueContainer item = pollWithMetadata(); + return item != null ? item.value : null; } - + @Override public E take() throws InterruptedException { - ValueWithMetadata item = takeWithMetadata(); - return item!=null ? item.value : null; + ValueContainer item = takeWithMetadata(); + return item != null ? item.value : null; } @Override public E poll(long timeout, TimeUnit unit) throws InterruptedException { - ValueWithMetadata item = pollWithMetadata(timeout, unit); - return item!=null ? item.value : null; + ValueContainer item = pollWithMetadata(timeout, unit); + return item != null ? item.value : null; } @Override @@ -357,7 +352,6 @@ public int size() { } } - @Override public void clear() { Connection c; @@ -375,7 +369,7 @@ public void clear() { throw new RuntimeException(e); } } - + /** * Legacy cleanupAll * @@ -396,7 +390,6 @@ public void cleanupAll() throws SQLException { cleanupAll(10); } - @Override public void cleanup(int days) throws SQLException { Connection c = ds.getConnection(); @@ -411,7 +404,6 @@ public void cleanup(int days) throws SQLException { } } - @Override public void cleanupAll(int days) throws SQLException { Connection c = ds.getConnection(); @@ -431,7 +423,8 @@ public void cleanupAll(int days) throws SQLException { * @param statement * @throws SQLException */ - abstract protected void setAddParameters(E value, PreparedStatement statement) throws SQLException; + abstract protected void setAddParameters(E value, int priority, PreparedStatement statement) + throws SQLException; /** * Binds table name to query @@ -473,6 +466,22 @@ protected E getValueFromResult(ResultSet rs, int index) throws SQLException { } } + /** + * Crates ValueContainer form result set. + * @param rs + * @return + * @throws SQLException + */ + protected ValueContainer valueContainerFromResult(ResultSet rs) throws SQLException { + //id, status, priority, value + return new ValueContainer( + rs.getLong(1), + rs.getString(2), + rs.getLong(3), + getValueFromResult(rs, 4) + ); + } + /** * Sets value to statement. If defined, serializator is used, otherwise setObject with type. * @param s @@ -489,7 +498,6 @@ protected void setValueToStatment(PreparedStatement s, int index, E obj) } } - /** * Returns sql query for add operation with binded table name * @return sql @@ -497,7 +505,7 @@ protected void setValueToStatment(PreparedStatement s, int index, E obj) protected String getAddQuery() { return setTable(addQuery); } - + /** * Returns sql query for add operation with binded table name * @return sql @@ -512,12 +520,12 @@ protected String getPeekQuery() { */ protected String[] getPollQuery() { String[] queries = new String[pollQuery.length]; - for(int i=0; i - * CREATE TABLE queue ( - * id INT UNSIGNED NOT NULL AUTO_INCREMENT, - * queue_name VARCHAR(255) NOT NULL, -- Queue name - * inserted TIMESTAMP NOT NULL, -- Time the row was inserted - * inserted_by VARCHAR(255) NOT NULL, -- and by who - * acquired TIMESTAMP NULL, -- Time the row was acquired - * acquired_by VARCHAR(255) NULL, -- and by who - * delayed_to TIMESTAMP NULL, -- Task delayed to - * value BLOB NOT NULL, -- The actual data - * status VARCHAR(255) NOT NULL DEFAULT 'NEW' -- The actual data - * PRIMARY KEY (id) - * ) ENGINE=INNODB DEFAULT CHARSET=UTF8; + * CREATE TABLE IF NOT EXISTS queue ( + * id int(10) unsigned NOT NULL AUTO_INCREMENT, + * queue_name varchar(255) NOT NULL, -- Queue name + * inserted timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, -- Time the row was inserted + * inserted_by varchar(255) NOT NULL, -- and by who + * acquired timestamp NULL DEFAULT NULL, -- Time the row was acquired + * acquired_by varchar(255) DEFAULT NULL, -- and by who + * status varchar(255) NOT NULL DEFAULT 'NEW', -- Item status + * delayed_to timestamp NULL DEFAULT NULL, -- Task delayed to + * priority int(11) NOT NULL DEFAULT '0', -- Item priority + * value blob NOT NULL, -- The actual data + * PRIMARY KEY (id) + * ) ENGINE=InnoDB DEFAULT CHARSET=utf8; *

* TODO Create efficient drainTo * @@ -42,16 +43,18 @@ public class MySQLBasedDelayQueue extends + " WHERE acquired IS NULL AND queue_name = ?"; protected String delayCondition = "AND (delayed_to<=NOW() OR delayed_to is null) "; + { addQuery = "INSERT INTO "+tableNamePlaceholder+" " - + "(queue_name, inserted, inserted_by, delayed_to, value) values " - + "(?, now(), ?, DATE_ADD(NOW(), INTERVAL ? SECOND), ?)"; + + "(queue_name, inserted, inserted_by, delayed_to, priority, value) values " + + "(?, now(), ?, DATE_ADD(NOW(), INTERVAL ? SECOND), ?, ?)"; - peekQuery = "SELECT id, status, value FROM "+tableNamePlaceholder+" WHERE " + peekQuery = "SELECT id, status, priority, value FROM "+tableNamePlaceholder+" WHERE " + "acquired IS NULL " + delayCondition + "AND queue_name = ? " - + "ORDER BY id ASC LIMIT 1"; + + "ORDER BY priority DESC, id ASC " + + "LIMIT 1; "; pollQuery = new String[] { "SET @update_id := -1; ", @@ -60,8 +63,10 @@ public class MySQLBasedDelayQueue extends + " acquired = NOW(), " + " acquired_by = ? " + "WHERE " + "acquired IS NULL " + delayCondition - + "AND queue_name = ? " + "ORDER BY id ASC " + "LIMIT 1; ", - "SELECT id, status, value FROM "+tableNamePlaceholder+" WHERE id = @update_id" + + "AND queue_name = ? " + + "ORDER BY priority DESC, id ASC " + + "LIMIT 1; ", + "SELECT id, status, priority, value FROM "+tableNamePlaceholder+" WHERE id = @update_id" }; } @@ -80,12 +85,12 @@ public MySQLBasedDelayQueue(DataSource ds, String queueTableName, @Override - protected void setAddParameters(E value, PreparedStatement s) - throws SQLException { + protected void setAddParameters(E value, int priority, PreparedStatement s) throws SQLException { s.setString(1, queueName); s.setObject(2, me); // Inserted by me s.setLong(3, value.getDelay(TimeUnit.SECONDS)); - setValueToStatment(s, 4, value); + s.setLong(4, priority); + setValueToStatment(s, 5, value); } /** diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java index 534427d..2517696 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java @@ -7,23 +7,21 @@ import net.bramp.serializator.Serializator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * A queue backed by MySQL *

- * CREATE TABLE queue ( - * id INT UNSIGNED NOT NULL AUTO_INCREMENT, - * queue_name VARCHAR(255) NOT NULL, -- Queue name - * inserted TIMESTAMP NOT NULL, -- Time the row was inserted - * inserted_by VARCHAR(255) NOT NULL, -- and by who - * acquired TIMESTAMP NULL, -- Time the row was acquired - * acquired_by VARCHAR(255) NULL, -- and by who - * value BLOB NOT NULL, -- The actual data - * status VARCHAR(255) NOT NULL DEFAULT 'NEW' -- The actual data - * PRIMARY KEY (id) - * ) ENGINE=INNODB DEFAULT CHARSET=UTF8; + * CREATE TABLE IF NOT EXISTS queue ( + * id int(10) unsigned NOT NULL AUTO_INCREMENT, + * queue_name varchar(255) NOT NULL, -- Queue name + * inserted timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, -- Time the row was inserted + * inserted_by varchar(255) NOT NULL, -- and by who + * acquired timestamp NULL DEFAULT NULL, -- Time the row was acquired + * acquired_by varchar(255) DEFAULT NULL, -- and by who + * status varchar(255) NOT NULL DEFAULT 'NEW', -- Item status + * priority int(11) NOT NULL DEFAULT '0', -- Item priority + * value blob NOT NULL, -- The actual data + * PRIMARY KEY (id) + * ) ENGINE=InnoDB DEFAULT CHARSET=utf8; *

* TODO Create efficient drainTo * @@ -33,12 +31,13 @@ public class MySQLBasedQueue extends AbstractMySQLQueue { { addQuery = "INSERT INTO "+tableNamePlaceholder+" " - + "(queue_name, inserted, inserted_by, value) values " - + "(?, now(), ?, ?)"; - peekQuery = "SELECT id, status, value FROM "+tableNamePlaceholder+" WHERE " + + "(queue_name, inserted, inserted_by, priority, value) values " + + "(?, now(), ?, ?, ?)"; + peekQuery = "SELECT id, status, priority, value FROM "+tableNamePlaceholder+" WHERE " + "acquired IS NULL " + "AND queue_name = ? " - + "ORDER BY id ASC LIMIT 1"; + + "ORDER BY priority DESC, id ASC " + + "LIMIT 1; "; pollQuery = new String[] { "SET @update_id := -1; ", "UPDATE "+tableNamePlaceholder+" SET " @@ -48,9 +47,9 @@ public class MySQLBasedQueue extends AbstractMySQLQueue { + "WHERE " + "acquired IS NULL " + "AND queue_name = ? " - + "ORDER BY id ASC " + + "ORDER BY priority DESC, id ASC " + "LIMIT 1; ", - "SELECT id, status, value FROM "+tableNamePlaceholder+" WHERE id = @update_id" + "SELECT id, status, priority, value FROM "+tableNamePlaceholder+" WHERE id = @update_id" }; } @@ -77,9 +76,10 @@ public MySQLBasedQueue(DataSource ds, String queueTableName, String queueName, S } @Override - protected void setAddParameters(E value, PreparedStatement s) throws SQLException { + protected void setAddParameters(E value, int priority, PreparedStatement s) throws SQLException { s.setString(1, queueName); s.setObject(2, me); // Inserted by me - setValueToStatment(s, 3, value); + s.setLong(3, priority); // Inserted by me + setValueToStatment(s, 4, value); } } diff --git a/src/main/java/net/bramp/db_patterns/queues/StatusableQueue.java b/src/main/java/net/bramp/db_patterns/queues/StatusableQueue.java deleted file mode 100644 index b00f140..0000000 --- a/src/main/java/net/bramp/db_patterns/queues/StatusableQueue.java +++ /dev/null @@ -1,27 +0,0 @@ -package net.bramp.db_patterns.queues; - -import java.util.concurrent.TimeUnit; - -public interface StatusableQueue { - public ValueWithMetadata pollWithMetadata(); - public ValueWithMetadata pollWithMetadata(long timeout, TimeUnit unit) throws InterruptedException; - public ValueWithMetadata peekWithMetadata(); - public ValueWithMetadata takeWithMetadata() throws InterruptedException; - public void updateStatus(long id, String newStatus); - public String getStatus(long id); - - public static class ValueWithMetadata { - public final long id; - public final String status; - public final E value; - ValueWithMetadata(long id, String status, E value) { - this.id = id; - this.status = status; - this.value = value; - } - @Override - public String toString() { - return "ValueWithMetadata["+id+" "+status+" "+value+"]"; - } - } -} diff --git a/src/main/java/net/bramp/db_patterns/queues/ValueContainer.java b/src/main/java/net/bramp/db_patterns/queues/ValueContainer.java new file mode 100644 index 0000000..fc16607 --- /dev/null +++ b/src/main/java/net/bramp/db_patterns/queues/ValueContainer.java @@ -0,0 +1,44 @@ +package net.bramp.db_patterns.queues; + +import net.bramp.db_patterns.queues.interfaces.ValueWithMetadata; +import net.bramp.db_patterns.queues.interfaces.ValueWithPriority; + +public class ValueContainer implements ValueWithMetadata, ValueWithPriority { + + public static final int DEFAULT_PRIORRITY = 0; + + protected long id; + protected String status; + protected long priority; + protected E value; + + ValueContainer(long id, String status, long priority, E value) { + this.id = id; + this.status = status; + this.priority = priority; + this.value = value; + } + ValueContainer(long id, String status, E value) { + this(id, status, DEFAULT_PRIORRITY, value); + } + @Override + public long getId() { + return id; + } + @Override + public String getStatus() { + return status; + } + @Override + public long getPriority() { + return priority; + } + @Override + public E getValue() { + return value; + } + @Override + public String toString() { + return "ValueWithMetadata["+id+" "+status+" "+value+"]"; + } +} \ No newline at end of file diff --git a/src/main/java/net/bramp/db_patterns/queues/CleanableQueue.java b/src/main/java/net/bramp/db_patterns/queues/interfaces/CleanableQueue.java similarity index 90% rename from src/main/java/net/bramp/db_patterns/queues/CleanableQueue.java rename to src/main/java/net/bramp/db_patterns/queues/interfaces/CleanableQueue.java index 0e5e0e3..513d831 100644 --- a/src/main/java/net/bramp/db_patterns/queues/CleanableQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/interfaces/CleanableQueue.java @@ -1,4 +1,4 @@ -package net.bramp.db_patterns.queues; +package net.bramp.db_patterns.queues.interfaces; import java.sql.SQLException; diff --git a/src/main/java/net/bramp/db_patterns/queues/interfaces/PriorityQueue.java b/src/main/java/net/bramp/db_patterns/queues/interfaces/PriorityQueue.java new file mode 100644 index 0000000..96950e3 --- /dev/null +++ b/src/main/java/net/bramp/db_patterns/queues/interfaces/PriorityQueue.java @@ -0,0 +1,5 @@ +package net.bramp.db_patterns.queues.interfaces; + +public interface PriorityQueue { + public boolean add(E value, int priority); +} diff --git a/src/main/java/net/bramp/db_patterns/queues/interfaces/StatusableQueue.java b/src/main/java/net/bramp/db_patterns/queues/interfaces/StatusableQueue.java new file mode 100644 index 0000000..c4d7d79 --- /dev/null +++ b/src/main/java/net/bramp/db_patterns/queues/interfaces/StatusableQueue.java @@ -0,0 +1,12 @@ +package net.bramp.db_patterns.queues.interfaces; + +import java.util.concurrent.TimeUnit; + +public interface StatusableQueue> { + public V pollWithMetadata(); + public V pollWithMetadata(long timeout, TimeUnit unit) throws InterruptedException; + public V peekWithMetadata(); + public V takeWithMetadata() throws InterruptedException; + public void updateStatus(long id, String newStatus); + public String getStatus(long id); +} diff --git a/src/main/java/net/bramp/db_patterns/queues/interfaces/ValueWithMetadata.java b/src/main/java/net/bramp/db_patterns/queues/interfaces/ValueWithMetadata.java new file mode 100644 index 0000000..78a677a --- /dev/null +++ b/src/main/java/net/bramp/db_patterns/queues/interfaces/ValueWithMetadata.java @@ -0,0 +1,7 @@ +package net.bramp.db_patterns.queues.interfaces; + +public interface ValueWithMetadata { + public long getId(); + public String getStatus(); + public E getValue(); +} \ No newline at end of file diff --git a/src/main/java/net/bramp/db_patterns/queues/interfaces/ValueWithPriority.java b/src/main/java/net/bramp/db_patterns/queues/interfaces/ValueWithPriority.java new file mode 100644 index 0000000..b945b37 --- /dev/null +++ b/src/main/java/net/bramp/db_patterns/queues/interfaces/ValueWithPriority.java @@ -0,0 +1,5 @@ +package net.bramp.db_patterns.queues.interfaces; + +public interface ValueWithPriority { + public long getPriority(); +} \ No newline at end of file diff --git a/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java b/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java index d70b4a8..dbe1850 100644 --- a/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java +++ b/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java @@ -1,16 +1,18 @@ package net.bramp.db_patterns.queues; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import java.sql.SQLException; import java.util.Arrays; import java.util.Collection; +import java.util.Random; import java.util.concurrent.TimeUnit; import javax.sql.DataSource; import net.bramp.db_patterns.DatabaseUtils; -import net.bramp.db_patterns.queues.StatusableQueue.ValueWithMetadata; import net.bramp.serializator.DefaultSerializator; import org.junit.After; @@ -109,10 +111,10 @@ public void statusTest() { Object a = valueFactory.apply("A"); assertTrue(queue.add(a)); - ValueWithMetadata v = queue.peekWithMetadata(); - queue.updateStatus(v.id, "Test1"); + ValueContainer v = queue.peekWithMetadata(); + queue.updateStatus(v.getId(), "Test1"); assertEquals(queue.getStatus(v.id), "Test1"); - queue.updateStatus(v.id, "Test2"); + queue.updateStatus(v.getId(), "Test2"); assertEquals(queue.getStatus(v.id), "Test2"); } @@ -146,4 +148,42 @@ public void pollBlockingTest() throws InterruptedException { assertEmpty(); } + + @Test + public void priorityTest() { + assertEmpty(); + + Object a = valueFactory.apply("A"); + Object b = valueFactory.apply("B"); + Object c = valueFactory.apply("C"); + Object d = valueFactory.apply("D"); + assertTrue(queue.add(a, 1)); + assertTrue(queue.add(b, 0)); + assertTrue(queue.add(c, 2)); + assertTrue(queue.add(d, 3)); + + assertEquals("Queue head should be D", d, queue.peek()); + assertEquals("Queue head should be D", d, queue.poll()); + assertEquals("Queue head should be C", c, queue.poll()); + assertEquals("Queue head should be A", a, queue.poll()); + assertEquals("Queue head should be B", b, queue.poll()); + + assertEmpty(); + } + @Test + public void priorityRandomTest() { + assertEmpty(); + + Random r = new Random(); + for(int i=0; i<100; i++) { + Object s = valueFactory.apply("Test str"); + assertTrue(queue.add(s, r.nextInt())); + } + + long prevPriority = Long.MAX_VALUE; + ValueContainer vc; + while((vc = queue.pollWithMetadata())!=null) { + assertTrue("Next priority should be <= previous", vc.getPriority()<=prevPriority); + } + } } diff --git a/src/test/java/net/bramp/db_patterns/queues/DelayedString.java b/src/test/java/net/bramp/db_patterns/queues/DelayedString.java index b539e24..e675222 100644 --- a/src/test/java/net/bramp/db_patterns/queues/DelayedString.java +++ b/src/test/java/net/bramp/db_patterns/queues/DelayedString.java @@ -49,6 +49,11 @@ public boolean equals(Object o) { public int hashCode() { return get().hashCode(); } + + @Override + public String toString() { + return get(); + } private long nowInSeconds() { return System.currentTimeMillis()/1000; diff --git a/src/test/java/net/bramp/db_patterns/queues/MultithreadMySQLBasedQueueTests.java b/src/test/java/net/bramp/db_patterns/queues/MultithreadMySQLBasedQueueTests.java index e9fcdb5..8d93f36 100644 --- a/src/test/java/net/bramp/db_patterns/queues/MultithreadMySQLBasedQueueTests.java +++ b/src/test/java/net/bramp/db_patterns/queues/MultithreadMySQLBasedQueueTests.java @@ -24,6 +24,8 @@ public class MultithreadMySQLBasedQueueTests { + private final String TABLE_NAME = "queue"; + final static Logger LOG = LoggerFactory.getLogger(MultithreadMySQLBasedQueueTests.class); private String queueName; @@ -55,7 +57,7 @@ public void setup() { ds = DatabaseUtils.createDataSource(); me = DatabaseUtils.getHostname(); - queue = new MySQLBasedQueue(ds, queueName, String.class, me); + queue = new MySQLBasedQueue(ds, TABLE_NAME, queueName, String.class, me); executor = Executors.newCachedThreadPool(); } @@ -63,7 +65,7 @@ public void setup() { @After public void cleanupDatabase() throws SQLException { queue.clear(); - queue.cleanupAll(); + queue.cleanupAll(10); assertEmpty(); } diff --git a/src/test/java/net/bramp/db_patterns/queues/StressMySQLBasedQueueTests.java b/src/test/java/net/bramp/db_patterns/queues/StressMySQLBasedQueueTests.java index 3e4edae..161bcad 100644 --- a/src/test/java/net/bramp/db_patterns/queues/StressMySQLBasedQueueTests.java +++ b/src/test/java/net/bramp/db_patterns/queues/StressMySQLBasedQueueTests.java @@ -27,6 +27,8 @@ */ public class StressMySQLBasedQueueTests { + private final String TABLE_NAME = "queue"; + final static Logger LOG = LoggerFactory.getLogger(StressMySQLBasedQueueTests.class); private String queueName; @@ -99,7 +101,7 @@ public void setup() { ds = DatabaseUtils.createDataSource(); me = DatabaseUtils.getHostname(); - queue = new MySQLBasedQueue(ds, queueName, Integer.class, me); + queue = new MySQLBasedQueue(ds, TABLE_NAME, queueName, Integer.class, me); executor = Executors.newCachedThreadPool(); } @@ -107,7 +109,7 @@ public void setup() { @After public void cleanupDatabase() throws SQLException { queue.clear(); - queue.cleanupAll(); + queue.cleanupAll(10); assertEmpty(); } From 6e6c57b7a7c1fcd86c9e35350c9ffd553da4dff2 Mon Sep 17 00:00:00 2001 From: Matzz Date: Tue, 12 Aug 2014 11:54:42 +0200 Subject: [PATCH 21/29] Reverted pom --- pom.xml | 312 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 165 insertions(+), 147 deletions(-) diff --git a/pom.xml b/pom.xml index 526e901..c118115 100644 --- a/pom.xml +++ b/pom.xml @@ -1,101 +1,119 @@ - 4.0.0 - - matzz - db-patterns - 0.3.4 - jar - - DB Patterns - Some simple DB patterns implemented onto of MySQL - https://github.com/bramp/db-patterns - - - - bramp - Andrew Brampton - - - - - - The BSD 2-Clause License - http://opensource.org/licenses/BSD-2-Clause - repo - - - - - UTF-8 - UTF-8 - - 0.9.30 - - - - - org.slf4j - slf4j-api - 1.6.2 - - - com.google.code.findbugs - jsr305 - 2.0.2 - provided - - - net.sourceforge.findbugs - annotations - 1.3.2 - provided - - - - mysql - mysql-connector-java - 5.1.31 - - - - junit - junit - 4.11 - test - - - - ch.qos.logback - logback-core - ${logback.version} - test - - - ch.qos.logback - logback-classic - ${logback.version} - test - - - com.google.guava - guava - 15.0 - test - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.6 - 1.6 - UTF-8 - - + 4.0.0 + + net.bramp.db-patterns + db-patterns + 0.2-SNAPSHOT + jar + + DB Patterns + Some simple DB patterns implemented onto of MySQL + https://github.com/bramp/db-patterns + + + + bramp + Andrew Brampton + + + + + + The BSD 2-Clause License + http://opensource.org/licenses/BSD-2-Clause + repo + + + + + https://github.com/bramp/db-patterns + scm:git:git@github.com:bramp/db-patterns.git + HEAD + + + + UTF-8 + UTF-8 + + 0.9.30 + + + + + org.slf4j + slf4j-api + 1.6.2 + + + com.google.code.findbugs + jsr305 + 2.0.2 + provided + + + net.sourceforge.findbugs + annotations + 1.3.2 + provided + + + + mysql + mysql-connector-java + 5.1.27 + provided + + + + junit + junit + 4.11 + test + + + + ch.qos.logback + logback-core + ${logback.version} + test + + + ch.qos.logback + logback-classic + ${logback.version} + test + + + com.google.guava + guava + 15.0 + test + + + + + + ossrh + https://oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.6 + 1.6 + UTF-8 + + org.apache.maven.plugins @@ -138,53 +156,53 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - install - - + + org.apache.maven.plugins + maven-deploy-plugin + 2.8.1 + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.5 + + + sign-artifacts + verify + + sign + + + + + + + org.apache.maven.plugins + maven-release-plugin + 2.5 + + true + false + release + deploy nexus-staging:release + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.1 + true + + ossrh + https://oss.sonatype.org/ + true + + + + + + install + + \ No newline at end of file From 32373bb4629d6fda0f28cccfc2ddc70f6649682c Mon Sep 17 00:00:00 2001 From: Matzz Date: Thu, 18 Sep 2014 21:56:18 +0200 Subject: [PATCH 22/29] Internally inverted priorities to enable sort index. --- README.md | 2 +- pom.xml | 2 +- .../db_patterns/queues/MySQLBasedDelayQueue.java | 12 +++++++----- .../bramp/db_patterns/queues/MySQLBasedQueue.java | 14 ++++++++------ .../queues/AbstractMySQLBasedQueueTest.java | 13 +++++++++++++ 5 files changed, 30 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index c092b42..c1f9c8d 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ Where V is class implementing ValueWithMetadata interface. ValueWithMetadata con Priority ----------------- MySQLBasedQueue, MySQLBasedDelayQueue implements PriorityQueue interface which enables setting items priority. The higher priority is, the earlier item will be polled from queue. - +From 0.3.5 version, due to lack of DESC index in mysql, internally in table values are stored inverted. While migrating to version 0.3.5 all priorities should be multiplied by -1. StatusableQueue brings such methods: ```java diff --git a/pom.xml b/pom.xml index 526e901..464f96f 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ matzz db-patterns - 0.3.4 + 0.3.5 jar DB Patterns diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java index 5486104..8c3b14c 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java @@ -29,6 +29,8 @@ * priority int(11) NOT NULL DEFAULT '0', -- Item priority * value blob NOT NULL, -- The actual data * PRIMARY KEY (id) + * UNIQUE KEY `queue_peek_index` (`acquired`,`queue_name`,`id`) + * UNIQUE KEY `sort_index` (`priority`,`id`) * ) ENGINE=InnoDB DEFAULT CHARSET=utf8; *

* TODO Create efficient drainTo @@ -47,13 +49,13 @@ public class MySQLBasedDelayQueue extends { addQuery = "INSERT INTO "+tableNamePlaceholder+" " + "(queue_name, inserted, inserted_by, delayed_to, priority, value) values " - + "(?, now(), ?, DATE_ADD(NOW(), INTERVAL ? SECOND), ?, ?)"; + + "(?, now(), ?, DATE_ADD(NOW(), INTERVAL ? SECOND), -?, ?)"; - peekQuery = "SELECT id, status, priority, value FROM "+tableNamePlaceholder+" WHERE " + peekQuery = "SELECT id, status, -priority, value FROM "+tableNamePlaceholder+" WHERE " + "acquired IS NULL " + delayCondition + "AND queue_name = ? " - + "ORDER BY priority DESC, id ASC " + + "ORDER BY priority ASC, id ASC " + "LIMIT 1; "; pollQuery = new String[] { @@ -64,9 +66,9 @@ public class MySQLBasedDelayQueue extends + " acquired_by = ? " + "WHERE " + "acquired IS NULL " + delayCondition + "AND queue_name = ? " - + "ORDER BY priority DESC, id ASC " + + "ORDER BY priority ASC, id ASC " + "LIMIT 1; ", - "SELECT id, status, priority, value FROM "+tableNamePlaceholder+" WHERE id = @update_id" + "SELECT id, status, -priority, value FROM "+tableNamePlaceholder+" WHERE id = @update_id" }; } diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java index 2517696..ad2486a 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java @@ -19,8 +19,10 @@ * acquired_by varchar(255) DEFAULT NULL, -- and by who * status varchar(255) NOT NULL DEFAULT 'NEW', -- Item status * priority int(11) NOT NULL DEFAULT '0', -- Item priority - * value blob NOT NULL, -- The actual data + * value blob NOT NULL, -- The actual data * PRIMARY KEY (id) + * UNIQUE KEY `queue_peek_index` (`acquired`,`queue_name`,`id`) + * UNIQUE KEY `sort_index` (`priority`,`id`) * ) ENGINE=InnoDB DEFAULT CHARSET=utf8; *

* TODO Create efficient drainTo @@ -32,11 +34,11 @@ public class MySQLBasedQueue extends AbstractMySQLQueue { { addQuery = "INSERT INTO "+tableNamePlaceholder+" " + "(queue_name, inserted, inserted_by, priority, value) values " - + "(?, now(), ?, ?, ?)"; - peekQuery = "SELECT id, status, priority, value FROM "+tableNamePlaceholder+" WHERE " + + "(?, now(), ?, -?, ?)"; + peekQuery = "SELECT id, status, -priority, value FROM "+tableNamePlaceholder+" WHERE " + "acquired IS NULL " + "AND queue_name = ? " - + "ORDER BY priority DESC, id ASC " + + "ORDER BY priority ASC, id ASC " + "LIMIT 1; "; pollQuery = new String[] { "SET @update_id := -1; ", @@ -47,9 +49,9 @@ public class MySQLBasedQueue extends AbstractMySQLQueue { + "WHERE " + "acquired IS NULL " + "AND queue_name = ? " - + "ORDER BY priority DESC, id ASC " + + "ORDER BY priority ASC, id ASC " + "LIMIT 1; ", - "SELECT id, status, priority, value FROM "+tableNamePlaceholder+" WHERE id = @update_id" + "SELECT id, status, -priority, value FROM "+tableNamePlaceholder+" WHERE id = @update_id" }; } diff --git a/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java b/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java index dbe1850..898e0d9 100644 --- a/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java +++ b/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java @@ -186,4 +186,17 @@ public void priorityRandomTest() { assertTrue("Next priority should be <= previous", vc.getPriority()<=prevPriority); } } + + @Test + public void getPriorityTest() { + assertEmpty(); + + queue.add(valueFactory.apply("a"), 11); + queue.add(valueFactory.apply("b"), 10); + queue.add(valueFactory.apply("c"), 12); + assertTrue(queue.pollWithMetadata().priority == 12); + assertTrue(queue.pollWithMetadata().priority == 11); + assertTrue(queue.pollWithMetadata().priority == 10); + + } } From 3e7a70a2ae3ac19ccb6ee4b832a29d47c86ec283 Mon Sep 17 00:00:00 2001 From: Matzz Date: Mon, 22 Sep 2014 11:35:09 +0200 Subject: [PATCH 23/29] Indexes fix --- .../net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java | 3 +-- .../java/net/bramp/db_patterns/queues/MySQLBasedQueue.java | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java index 8c3b14c..aba8702 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java @@ -29,8 +29,7 @@ * priority int(11) NOT NULL DEFAULT '0', -- Item priority * value blob NOT NULL, -- The actual data * PRIMARY KEY (id) - * UNIQUE KEY `queue_peek_index` (`acquired`,`queue_name`,`id`) - * UNIQUE KEY `sort_index` (`priority`,`id`) + * UNIQUE KEY `queue_peek_index` (`acquired`,`queue_name`, `delayed_to`, `priority`,`id`) * ) ENGINE=InnoDB DEFAULT CHARSET=utf8; *

* TODO Create efficient drainTo diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java index ad2486a..50729ae 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java @@ -21,8 +21,7 @@ * priority int(11) NOT NULL DEFAULT '0', -- Item priority * value blob NOT NULL, -- The actual data * PRIMARY KEY (id) - * UNIQUE KEY `queue_peek_index` (`acquired`,`queue_name`,`id`) - * UNIQUE KEY `sort_index` (`priority`,`id`) + * UNIQUE KEY `queue_peek_index` (`acquired`,`queue_name`, `priority`,`id`) * ) ENGINE=InnoDB DEFAULT CHARSET=utf8; *

* TODO Create efficient drainTo From 6846eb289d5078f78f1f59db9e761d18a9685205 Mon Sep 17 00:00:00 2001 From: Matzz Date: Wed, 24 Sep 2014 11:03:43 +0200 Subject: [PATCH 24/29] Enhanced queue pull performance. --- pom.xml | 2 +- .../queues/AbstractMySQLQueue.java | 4 +-- .../queues/MySQLBasedDelayQueue.java | 25 ++++++++++++------- .../db_patterns/queues/MySQLBasedQueue.java | 16 +++++++----- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/pom.xml b/pom.xml index 464f96f..980084b 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ matzz db-patterns - 0.3.5 + 0.3.6 jar DB Patterns diff --git a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java index 4700774..a47c7a9 100644 --- a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java @@ -189,8 +189,8 @@ public ValueContainer pollWithMetadata() { s1.execute(); PreparedStatement s2 = c.prepareStatement(pollQuery[1]); - s2.setString(1, me); // Acquired by me - s2.setString(2, queueName); + s2.setString(1, queueName); + s2.setString(2, me); // Acquired by me s2.execute(); CallableStatement s3 = c.prepareCall(pollQuery[2]); diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java index aba8702..cc7fac2 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java @@ -57,17 +57,24 @@ public class MySQLBasedDelayQueue extends + "ORDER BY priority ASC, id ASC " + "LIMIT 1; "; + pollQuery = new String[] { "SET @update_id := -1; ", - "UPDATE "+tableNamePlaceholder+" SET " - + " id = (SELECT @update_id := id), " - + " acquired = NOW(), " - + " acquired_by = ? " - + "WHERE " + "acquired IS NULL " + delayCondition - + "AND queue_name = ? " - + "ORDER BY priority ASC, id ASC " - + "LIMIT 1; ", - "SELECT id, status, -priority, value FROM "+tableNamePlaceholder+" WHERE id = @update_id" + "UPDATE "+tableNamePlaceholder+" u " + + "join ( " + + "SELECT id from " + tableNamePlaceholder + " " + + "WHERE " + + "acquired IS NULL " + + delayCondition + + "AND queue_name = ? " + + "ORDER BY priority ASC, id ASC " + + "LIMIT 1) s " + + "ON u.id = s.id " + + "SET " + + "u.id = (SELECT @update_id := s.id), " + + "acquired = NOW(), " + + "acquired_by = ?; ", + "SELECT id, status, -priority, value FROM "+tableNamePlaceholder+" WHERE id = @update_id;" }; } diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java index 50729ae..317ca8c 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java @@ -41,16 +41,20 @@ public class MySQLBasedQueue extends AbstractMySQLQueue { + "LIMIT 1; "; pollQuery = new String[] { "SET @update_id := -1; ", - "UPDATE "+tableNamePlaceholder+" SET " - + " id = (SELECT @update_id := id), " - + " acquired = NOW(), " - + " acquired_by = ? " + "UPDATE "+tableNamePlaceholder+" u " + + "join ( " + + "SELECT id from " + tableNamePlaceholder + " " + "WHERE " + "acquired IS NULL " + "AND queue_name = ? " + "ORDER BY priority ASC, id ASC " - + "LIMIT 1; ", - "SELECT id, status, -priority, value FROM "+tableNamePlaceholder+" WHERE id = @update_id" + + "LIMIT 1) s " + + "ON u.id = s.id " + + "SET " + + "u.id = (SELECT @update_id := s.id), " + + "acquired = NOW(), " + + "acquired_by = ?; ", + "SELECT id, status, -priority, value FROM "+tableNamePlaceholder+" WHERE id = @update_id;" }; } From f4c4118797bf1add9d903acabf12edf01ed85fe8 Mon Sep 17 00:00:00 2001 From: Matzz Date: Thu, 25 Sep 2014 12:34:32 +0200 Subject: [PATCH 25/29] Handling deadlocks --- pom.xml | 3 +- .../queues/AbstractMySQLQueue.java | 107 ++++++++++++------ 2 files changed, 72 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 980084b..13de0d7 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ matzz db-patterns - 0.3.6 + 0.3.7 jar DB Patterns @@ -33,6 +33,7 @@ + org.slf4j slf4j-api diff --git a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java index a47c7a9..50ca8d9 100644 --- a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java @@ -7,10 +7,13 @@ import java.sql.SQLException; import java.util.Date; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Condition; import javax.sql.DataSource; +import com.mysql.jdbc.exceptions.jdbc4.MySQLTransactionRollbackException; + import net.bramp.db_patterns.locks.MySQLSleepBasedCondition; import net.bramp.db_patterns.queues.interfaces.CleanableQueue; import net.bramp.db_patterns.queues.interfaces.PriorityQueue; @@ -148,7 +151,7 @@ public boolean add(E value, int priority) { throw new RuntimeException(e); } } - + @Override public ValueContainer peekWithMetadata() { try { @@ -177,44 +180,77 @@ public ValueContainer peekWithMetadata() { } } - @Override - public ValueContainer pollWithMetadata() { - try { - Connection c = ds.getConnection(); - String[] pollQuery = getPollQuery(); - try { - c.setAutoCommit(false); + protected ValueContainer executePollWithMetadata(Connection c, + String[] pollQuery) throws SQLException { + PreparedStatement s0 = null; + PreparedStatement s1 = null; + PreparedStatement s2 = null; - CallableStatement s1 = c.prepareCall(pollQuery[0]); - s1.execute(); + try { + s0 = c.prepareStatement(pollQuery[0]); + s0.execute(); - PreparedStatement s2 = c.prepareStatement(pollQuery[1]); - s2.setString(1, queueName); - s2.setString(2, me); // Acquired by me - s2.execute(); + s1 = c.prepareStatement(pollQuery[1]); + s1.setString(1, queueName); + s1.setString(2, me); // Acquired by me + s1.execute(); - CallableStatement s3 = c.prepareCall(pollQuery[2]); - s3.execute(); + s2 = c.prepareStatement(pollQuery[2]); + boolean success = s2.execute(); - c.commit(); + c.commit(); - if (s3.execute()) { - ResultSet rs = s3.getResultSet(); - if (rs != null && rs.next()) { - return valueContainerFromResult(rs); - } + if (success) { + ResultSet rs = s2.getResultSet(); + if (rs != null && rs.next()) { + return valueContainerFromResult(rs); } + } + return null; - return null; + } finally { + try { if (s0 != null) s0.close(); } catch (Exception e) { } + try { if (s1 != null) s0.close(); } catch (Exception e) { } + try { if (s2 != null) s0.close(); } catch (Exception e) { } + } + } + + AtomicInteger eCnt = new AtomicInteger(0); - } finally { - c.setAutoCommit(true); - c.close(); + @Override + public ValueContainer pollWithMetadata() { + String[] pollQuery = getPollQuery(); + Connection c = null; + try { + c = ds.getConnection(); + c.setAutoCommit(false); + SQLException lastException = null; + do { + try { + return executePollWithMetadata(c, pollQuery); + } + catch(SQLException e) { + c.rollback(); + lastException = e; + } } - + while(lastException instanceof MySQLTransactionRollbackException); + if(lastException!=null) { + throw lastException; + } + return null; } catch (SQLException e) { throw new RuntimeException(e); } + finally { + if (c != null) { + try { + c.setAutoCommit(true); + c.close(); + } catch (Exception ex) { + } + } + } } @Override @@ -423,8 +459,8 @@ public void cleanupAll(int days) throws SQLException { * @param statement * @throws SQLException */ - abstract protected void setAddParameters(E value, int priority, PreparedStatement statement) - throws SQLException; + abstract protected void setAddParameters(E value, int priority, + PreparedStatement statement) throws SQLException; /** * Binds table name to query @@ -472,14 +508,11 @@ protected E getValueFromResult(ResultSet rs, int index) throws SQLException { * @return * @throws SQLException */ - protected ValueContainer valueContainerFromResult(ResultSet rs) throws SQLException { - //id, status, priority, value - return new ValueContainer( - rs.getLong(1), - rs.getString(2), - rs.getLong(3), - getValueFromResult(rs, 4) - ); + protected ValueContainer valueContainerFromResult(ResultSet rs) + throws SQLException { + // id, status, priority, value + return new ValueContainer(rs.getLong(1), rs.getString(2), + rs.getLong(3), getValueFromResult(rs, 4)); } /** From edf7aec6c6509728172dcbde6d50faec04986205 Mon Sep 17 00:00:00 2001 From: Matzz Date: Thu, 25 Sep 2014 13:10:49 +0200 Subject: [PATCH 26/29] Rewrited sql to prevend deadlocks --- pom.xml | 2 +- .../db_patterns/queues/AbstractMySQLQueue.java | 12 +++++++----- .../db_patterns/queues/MySQLBasedDelayQueue.java | 16 +++++++--------- .../db_patterns/queues/MySQLBasedQueue.java | 15 +++++++-------- 4 files changed, 22 insertions(+), 23 deletions(-) diff --git a/pom.xml b/pom.xml index 13de0d7..802ac71 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ matzz db-patterns - 0.3.7 + 0.3.8 jar DB Patterns diff --git a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java index 50ca8d9..c6bc624 100644 --- a/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/AbstractMySQLQueue.java @@ -192,16 +192,18 @@ protected ValueContainer executePollWithMetadata(Connection c, s1 = c.prepareStatement(pollQuery[1]); s1.setString(1, queueName); - s1.setString(2, me); // Acquired by me - s1.execute(); + boolean success = s1.execute(); - s2 = c.prepareStatement(pollQuery[2]); - boolean success = s2.execute(); + if(success) { + s2 = c.prepareStatement(pollQuery[2]); + s2.setString(1, me); // Acquired by me + s2.execute(); + } c.commit(); if (success) { - ResultSet rs = s2.getResultSet(); + ResultSet rs = s1.getResultSet(); if (rs != null && rs.next()) { return valueContainerFromResult(rs); } diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java index cc7fac2..c5706b4 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java @@ -57,24 +57,22 @@ public class MySQLBasedDelayQueue extends + "ORDER BY priority ASC, id ASC " + "LIMIT 1; "; - pollQuery = new String[] { "SET @update_id := -1; ", - "UPDATE "+tableNamePlaceholder+" u " - + "join ( " - + "SELECT id from " + tableNamePlaceholder + " " + "SELECT (SELECT @update_id := id), status, -priority, value " + + "FROM "+tableNamePlaceholder+" " + "WHERE " + "acquired IS NULL " + delayCondition + "AND queue_name = ? " + "ORDER BY priority ASC, id ASC " - + "LIMIT 1) s " - + "ON u.id = s.id " + + "LIMIT 1 " + + "FOR UPDATE", + "UPDATE "+tableNamePlaceholder+" u " + "SET " - + "u.id = (SELECT @update_id := s.id), " + "acquired = NOW(), " - + "acquired_by = ?; ", - "SELECT id, status, -priority, value FROM "+tableNamePlaceholder+" WHERE id = @update_id;" + + "acquired_by = ? " + + "where u.id = @update_id;" }; } diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java index 317ca8c..041c60b 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedQueue.java @@ -41,20 +41,19 @@ public class MySQLBasedQueue extends AbstractMySQLQueue { + "LIMIT 1; "; pollQuery = new String[] { "SET @update_id := -1; ", - "UPDATE "+tableNamePlaceholder+" u " - + "join ( " - + "SELECT id from " + tableNamePlaceholder + " " + "SELECT (SELECT @update_id := id), status, -priority, value " + + "FROM "+tableNamePlaceholder+" " + "WHERE " + "acquired IS NULL " + "AND queue_name = ? " + "ORDER BY priority ASC, id ASC " - + "LIMIT 1) s " - + "ON u.id = s.id " + + "LIMIT 1 " + + "FOR UPDATE", + "UPDATE "+tableNamePlaceholder+" u " + "SET " - + "u.id = (SELECT @update_id := s.id), " + "acquired = NOW(), " - + "acquired_by = ?; ", - "SELECT id, status, -priority, value FROM "+tableNamePlaceholder+" WHERE id = @update_id;" + + "acquired_by = ? " + + "where u.id = @update_id;" }; } From d88270a12bf919864e5fa34395c3ab9aae542454 Mon Sep 17 00:00:00 2001 From: Matzz Date: Thu, 25 Sep 2014 13:38:08 +0200 Subject: [PATCH 27/29] version 0.3.9 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 802ac71..508feb1 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ matzz db-patterns - 0.3.8 + 0.3.9 jar DB Patterns From 90bb7516de4b1e83775091e06b4c6f528e2d31bc Mon Sep 17 00:00:00 2001 From: Matzz Date: Thu, 25 Sep 2014 14:28:38 +0200 Subject: [PATCH 28/29] Fixed leaking connections in delayed Queue. Added value uniqueness test in multithreaded environment. --- .../queues/MySQLBasedDelayQueue.java | 112 +++++++++--------- .../queues/AbstractMySQLBasedQueueTest.java | 73 ++++++++++-- 2 files changed, 122 insertions(+), 63 deletions(-) diff --git a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java index c5706b4..6dbf9b9 100644 --- a/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java +++ b/src/main/java/net/bramp/db_patterns/queues/MySQLBasedDelayQueue.java @@ -39,59 +39,51 @@ */ public class MySQLBasedDelayQueue extends AbstractMySQLQueue { - - protected String closestDelayQuery = "SELECT min(TIME_TO_SEC(TIMEDIFF(delayed_to,NOW()))) FROM " + tableNamePlaceholder + + protected String closestDelayQuery = "SELECT min(TIME_TO_SEC(TIMEDIFF(delayed_to,NOW()))) FROM " + + tableNamePlaceholder + " WHERE acquired IS NULL AND queue_name = ?"; protected String delayCondition = "AND (delayed_to<=NOW() OR delayed_to is null) "; - + { - addQuery = "INSERT INTO "+tableNamePlaceholder+" " + addQuery = "INSERT INTO " + + tableNamePlaceholder + + " " + "(queue_name, inserted, inserted_by, delayed_to, priority, value) values " + "(?, now(), ?, DATE_ADD(NOW(), INTERVAL ? SECOND), -?, ?)"; - - peekQuery = "SELECT id, status, -priority, value FROM "+tableNamePlaceholder+" WHERE " - + "acquired IS NULL " - + delayCondition - + "AND queue_name = ? " - + "ORDER BY priority ASC, id ASC " - + "LIMIT 1; "; + + peekQuery = "SELECT id, status, -priority, value FROM " + + tableNamePlaceholder + " WHERE " + "acquired IS NULL " + + delayCondition + "AND queue_name = ? " + + "ORDER BY priority ASC, id ASC " + "LIMIT 1; "; pollQuery = new String[] { "SET @update_id := -1; ", "SELECT (SELECT @update_id := id), status, -priority, value " - + "FROM "+tableNamePlaceholder+" " - + "WHERE " - + "acquired IS NULL " - + delayCondition - + "AND queue_name = ? " - + "ORDER BY priority ASC, id ASC " - + "LIMIT 1 " - + "FOR UPDATE", - "UPDATE "+tableNamePlaceholder+" u " - + "SET " - + "acquired = NOW(), " - + "acquired_by = ? " - + "where u.id = @update_id;" - }; + + "FROM " + tableNamePlaceholder + " " + "WHERE " + + "acquired IS NULL " + delayCondition + + "AND queue_name = ? " + + "ORDER BY priority ASC, id ASC " + "LIMIT 1 " + + "FOR UPDATE", + "UPDATE " + tableNamePlaceholder + " u " + "SET " + + "acquired = NOW(), " + "acquired_by = ? " + + "where u.id = @update_id;" }; } - - public MySQLBasedDelayQueue(DataSource ds, String queueTableName, String queueName, Class type, String me) { super(ds, queueTableName, queueName, type, me); } - public MySQLBasedDelayQueue(DataSource ds, String queueTableName, String queueName, Serializator serializator, String me) { super(ds, queueTableName, queueName, serializator, me); } - @Override - protected void setAddParameters(E value, int priority, PreparedStatement s) throws SQLException { + protected void setAddParameters(E value, int priority, PreparedStatement s) + throws SQLException { s.setString(1, queueName); s.setObject(2, me); // Inserted by me s.setLong(3, value.getDelay(TimeUnit.SECONDS)); @@ -106,28 +98,33 @@ protected void setAddParameters(E value, int priority, PreparedStatement s) thro */ protected long getClosestDelay() throws SQLException { int minDelay = 0; - + String query = setTable(closestDelayQuery); Connection c = ds.getConnection(); - - PreparedStatement s = c.prepareStatement(query); - s.setString(1, queueName); - if (s.execute()) { - ResultSet rs = s.getResultSet(); - if (rs != null && rs.next()) { - minDelay = rs.getInt(1); + try { + PreparedStatement s = c.prepareStatement(query); + s.setString(1, queueName); + if (s.execute()) { + ResultSet rs = s.getResultSet(); + if (rs != null && rs.next()) { + minDelay = rs.getInt(1); + } } + return minDelay; + } finally { + c.close(); } - return minDelay; } - protected ScheduledExecutorService wakeupScheduler = Executors.newScheduledThreadPool(1); + protected ScheduledExecutorService wakeupScheduler = Executors + .newScheduledThreadPool(1); protected ScheduledFuture wakeupTask = null; + protected class WakeupTask implements Runnable { @Override public void run() { condition.signal(); - synchronized(wakeupScheduler) { + synchronized (wakeupScheduler) { wakeupScheduler.schedule(new Runnable() { @Override public void run() { @@ -137,26 +134,29 @@ public void run() { } } }; - + @Override protected void wakeupThread() { synchronized (wakeupScheduler) { - long delaySeconds; - try { - delaySeconds = getClosestDelay(); - if(delaySeconds<=0) { - delaySeconds = 1; - } - } catch (SQLException e) { + long delaySeconds; + try { + delaySeconds = getClosestDelay(); + if (delaySeconds <= 0) { delaySeconds = 1; - e.printStackTrace(); - } - if(wakeupTask!=null && wakeupTask.getDelay(TimeUnit.SECONDS)>delaySeconds) { - wakeupTask.cancel(false); - } - if(wakeupTask==null || wakeupTask.isDone() || wakeupTask.isCancelled()) { - wakeupTask = wakeupScheduler.schedule(new WakeupTask(), delaySeconds, TimeUnit.SECONDS); } + } catch (SQLException e) { + delaySeconds = 1; + e.printStackTrace(); + } + if (wakeupTask != null + && wakeupTask.getDelay(TimeUnit.SECONDS) > delaySeconds) { + wakeupTask.cancel(false); + } + if (wakeupTask == null || wakeupTask.isDone() + || wakeupTask.isCancelled()) { + wakeupTask = wakeupScheduler.schedule(new WakeupTask(), + delaySeconds, TimeUnit.SECONDS); + } } } diff --git a/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java b/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java index 898e0d9..88f0390 100644 --- a/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java +++ b/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java @@ -5,9 +5,19 @@ import static org.junit.Assert.assertTrue; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; import java.util.Random; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import javax.sql.DataSource; @@ -56,8 +66,7 @@ public String apply(String input) { queueName, String.class, me); return Arrays.asList(new Object[][] { new Object[] { dsFactory, delayedQueue }, - new Object[] { sFactory, standardQueue } - }); + new Object[] { sFactory, standardQueue } }); } public AbstractMySQLBasedQueueTest(Function valueFactory, @@ -93,7 +102,7 @@ public void test() { assertTrue(queue.add(b)); assertEquals("Queue should start empty", 2, queue.size()); - + assertEquals("Queue head should be A", a, queue.peek()); assertEquals("Queue head should be A", a, queue.poll()); @@ -170,23 +179,25 @@ public void priorityTest() { assertEmpty(); } + @Test public void priorityRandomTest() { assertEmpty(); Random r = new Random(); - for(int i=0; i<100; i++) { + for (int i = 0; i < 100; i++) { Object s = valueFactory.apply("Test str"); assertTrue(queue.add(s, r.nextInt())); } long prevPriority = Long.MAX_VALUE; ValueContainer vc; - while((vc = queue.pollWithMetadata())!=null) { - assertTrue("Next priority should be <= previous", vc.getPriority()<=prevPriority); + while ((vc = queue.pollWithMetadata()) != null) { + assertTrue("Next priority should be <= previous", + vc.getPriority() <= prevPriority); } } - + @Test public void getPriorityTest() { assertEmpty(); @@ -197,6 +208,54 @@ public void getPriorityTest() { assertTrue(queue.pollWithMetadata().priority == 12); assertTrue(queue.pollWithMetadata().priority == 11); assertTrue(queue.pollWithMetadata().priority == 10); + + } + + @Test + public void multhreadUniqResultsTest() throws InterruptedException, + ExecutionException { + assertEmpty(); + + class Worker implements Callable> { + @Override + public List call() { + List done = new LinkedList(); + try { + Object last = null; + do { + last = queue.poll(5, TimeUnit.SECONDS); + if(last!=null) { + done.add(last); + } + } while (last != null); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return done; + } + + } + + int threadsCnt = 20; + int itemsCnt = 500; + for (int i = 0; i < itemsCnt; i++) { + assertTrue(queue.add(valueFactory.apply(String.valueOf(i)))); + } + ExecutorService executor = Executors.newFixedThreadPool(threadsCnt); + List>> futures = new ArrayList>>( + threadsCnt); + for (int i = 0; i < threadsCnt; i++) { + futures.add(executor.submit(new Worker())); + } + List allDoneList = new LinkedList(); + for (int i = 0; i < threadsCnt; i++) { + List currentDone = futures.get(i).get(); + System.out.println(currentDone); + allDoneList.addAll(currentDone); + } + Set uniqSet = new HashSet(allDoneList); + assertEquals(uniqSet.size(), allDoneList.size()); + assertEmpty(); } } From c4d17e6de29760d69d37f29dd581e6235b8e27df Mon Sep 17 00:00:00 2001 From: Matzz Date: Thu, 25 Sep 2014 14:34:49 +0200 Subject: [PATCH 29/29] Removed println --- .../bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java b/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java index 88f0390..6fe5583 100644 --- a/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java +++ b/src/test/java/net/bramp/db_patterns/queues/AbstractMySQLBasedQueueTest.java @@ -251,7 +251,6 @@ public List call() { List allDoneList = new LinkedList(); for (int i = 0; i < threadsCnt; i++) { List currentDone = futures.get(i).get(); - System.out.println(currentDone); allDoneList.addAll(currentDone); } Set uniqSet = new HashSet(allDoneList);