diff --git a/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/MergeLogic.java b/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/MergeLogic.java
new file mode 100644
index 0000000..120b378
--- /dev/null
+++ b/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/MergeLogic.java
@@ -0,0 +1,207 @@
+/**
+ *
+ * Copyright 2010 Paul Scherrer Institute. All rights reserved.
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This code is distributed in the hope that it will be useful,
+ * but without any warranty; without even the implied warranty of
+ * merchantability or fitness for a particular purpose. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this code. If not, see .
+ *
+ */
+
+package ch.psi.fda.core.loops.cr;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.google.common.eventbus.EventBus;
+import com.google.common.eventbus.Subscribe;
+
+import ch.psi.fda.messages.DataMessage;
+import ch.psi.fda.messages.EndOfStreamMessage;
+import ch.psi.fda.messages.Message;
+import ch.psi.fda.messages.Metadata;
+
+/**
+ * Class to merge two data streams into one. The secondary queues data is added to the primary queues data.
+ * The resulting queue therefor will hold the same amount of elements as the primary queue does.
+ * The datagroup flag set in the end of stream message will be set according to the flag set in the primary queue.
+ */
+public class MergeLogic {
+
+ private static final Logger logger = Logger.getLogger(MergeLogic.class.getName());
+
+ private final EventBus primaryEventBus;
+ private final EventBus secondaryEventBus;
+ private final EventBus outputEventBus;
+
+ private List metadata;
+
+ // Handler of the secondary event bus. We need to keep the reference on this
+ // to be able to unregister ...
+ private Object handler;
+
+
+ boolean firstPrimary = true;
+ boolean firstSecondary = true;
+
+ private BlockingQueue queue = new LinkedBlockingQueue<>();
+ private List currentData = null;
+
+
+ public MergeLogic(EventBus primaryEventBus, EventBus secondaryEventBus, EventBus outputEventBus){
+ this.primaryEventBus = primaryEventBus;
+ this.secondaryEventBus = secondaryEventBus;
+ this.outputEventBus = outputEventBus;
+ }
+
+ // Enables the merge logic
+ public void enable(){
+
+ firstPrimary = true;
+ firstSecondary = true;
+
+ handler = new Object(){
+ @Subscribe
+ public void onMessage(Message m){
+ // Only queue data messages
+ if(m instanceof DataMessage){
+ queue.add((DataMessage) m);
+ }
+ }
+ };
+
+ primaryEventBus.register(this);
+ secondaryEventBus.register(handler);
+ }
+
+ // Disables the merge logic
+ public void disable(){
+ primaryEventBus.unregister(this);
+ secondaryEventBus.unregister(handler);
+ }
+
+
+ /**
+ * This is the master merging logic
+ * @param m Data message from primary event bus
+ */
+ @Subscribe
+ public void onMessage(Message rawMessage){
+ if(rawMessage instanceof EndOfStreamMessage){
+ outputEventBus.post(rawMessage);
+
+ // Clear queue of secondary event bus
+ queue.clear();
+ return;
+ }
+ else if(!(rawMessage instanceof DataMessage)){
+ logger.warning("Message type not supported - ignore");
+ return;
+ }
+
+ DataMessage message = (DataMessage) rawMessage;
+
+ // Hack to remove the synchronization timestamp from the primary event bus
+ if(firstPrimary){
+ firstPrimary = false;
+
+ metadata = new ArrayList<>();
+ List pqm = message.getMetadata();
+ metadata.add(pqm.get(0)); // add first component (this is the actuator)
+ // Skip the next component as this is the timestamp used to merge the data
+ for(int i=2;i sqm = msCheck.getMetadata();
+ // Skip first two components of the message as this is the timestamp
+ for(int i=2;imilliseconds || (currMilliCheck==milliseconds && currNanoCheck>nanoOffset)){
+ break;
+ }
+
+ try {
+ DataMessage currentDataMessage = queue.take();
+ currentData = currentDataMessage.getData();
+ // Remove timestamps used for synchronization
+ currentData.remove(0);
+ currentData.remove(0);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ if(currentData == null){
+ throw new RuntimeException("No SCR data arrived");
+ }
+
+ // Add data to primary data queue message and put it into the out queue
+ message.getData().addAll(currentData);
+ message.setMetadata(metadata);
+ outputEventBus.post(message);
+ }
+}
diff --git a/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/ParallelCrlogic.java b/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/ParallelCrlogic.java
index a3f2037..cf840c9 100644
--- a/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/ParallelCrlogic.java
+++ b/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/ParallelCrlogic.java
@@ -21,24 +21,20 @@ package ch.psi.fda.core.loops.cr;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import com.google.common.eventbus.EventBus;
-import com.google.common.eventbus.Subscribe;
import ch.psi.fda.core.Action;
import ch.psi.fda.core.ActionLoop;
-import ch.psi.fda.messages.Message;
public class ParallelCrlogic implements ActionLoop {
@@ -62,7 +58,7 @@ public class ParallelCrlogic implements ActionLoop {
*/
private List postActions;
- private ParallelCrlogicStreamMerge merger;
+ private MergeLogic mergeLogic;
private final EventBus eventbus;
@@ -89,23 +85,9 @@ public class ParallelCrlogic implements ActionLoop {
this.preActions = new ArrayList();
this.postActions = new ArrayList();
- final BlockingQueue q1 = new LinkedBlockingQueue<>();
- final BlockingQueue q2 = new LinkedBlockingQueue<>();
- crlogic.getEventBus().register(new Object(){
- @Subscribe
- public void onMessage(Message m){
- q1.add(m);
- }
- });
- scrlogic.getEventBus().register(new Object(){
- @Subscribe
- public void onMessage(Message m){
- q2.add(m);
- }
- });
- this.merger = new ParallelCrlogicStreamMerge(q1, q2, eventbus);
+ this.mergeLogic = new MergeLogic(crlogic.getEventBus(), scrlogic.getEventBus(), eventbus);
}
@Override
@@ -115,6 +97,8 @@ public class ParallelCrlogic implements ActionLoop {
@Override
public void execute() throws InterruptedException {
+ mergeLogic.enable();
+
// Execute pre actions
for(Action action: preActions){
action.execute();
@@ -175,8 +159,6 @@ public class ParallelCrlogic implements ActionLoop {
}
}
- merger.merge();
-
// Wait until all threads have finished
service.shutdown();
service.awaitTermination(1, TimeUnit.MINUTES);
@@ -187,6 +169,7 @@ public class ParallelCrlogic implements ActionLoop {
action.execute();
}
+ mergeLogic.disable();
}
@Override
diff --git a/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/ParallelCrlogicStreamMerge.java b/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/ParallelCrlogicStreamMerge.java
deleted file mode 100644
index 9a5cc24..0000000
--- a/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/ParallelCrlogicStreamMerge.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/**
- *
- * Copyright 2010 Paul Scherrer Institute. All rights reserved.
- *
- * This code is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Lesser General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This code is distributed in the hope that it will be useful,
- * but without any warranty; without even the implied warranty of
- * merchantability or fitness for a particular purpose. See the
- * GNU Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this code. If not, see .
- *
- */
-
-package ch.psi.fda.core.loops.cr;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import com.google.common.eventbus.EventBus;
-
-import ch.psi.fda.messages.DataMessage;
-import ch.psi.fda.messages.EndOfStreamMessage;
-import ch.psi.fda.messages.Message;
-import ch.psi.fda.messages.Metadata;
-
-/**
- * Class to merge two data streams into one. The secondary queues data is added to the primary queues data.
- * The resulting queue therefor will hold the same amount of elements as the primary queue does.
- * The datagroup flag set in the end of stream message will be set according to the flag set in the primary queue.
- */
-public class ParallelCrlogicStreamMerge {
-
- private static final Logger logger = Logger.getLogger(ParallelCrlogicStreamMerge.class.getName());
-
- private BlockingQueue primaryQueue;
- private BlockingQueue secondaryQueue;
-
- private final EventBus eventbus;
-
- private List metadata;
-
- /**
- * @param pqueue Primary/master queue
- * @param squeue Secondary queue
- */
- public ParallelCrlogicStreamMerge(BlockingQueue pqueue, BlockingQueue squeue, EventBus ebus){
- this.primaryQueue = pqueue;
- this.secondaryQueue = squeue;
-
- this.eventbus = ebus;
- }
-
- /**
- * Merge the streams based on the timestamp
- * @throws InterruptedException
- */
- public void merge() throws InterruptedException{
-
- metadata = new ArrayList<>();
-
- boolean firstPrimary = true;
- boolean firstSecondary = true;
-
- // Actual data of the secondary queue
- List currData = null;
-
- // Take first element of the primary queue (wait until message is available)
- Message m = primaryQueue.take();
-
- while(! (m instanceof EndOfStreamMessage)){
-
- if(m instanceof DataMessage){
-
- DataMessage dm = (DataMessage) m;
-
- if(firstPrimary){
- firstPrimary = false;
- List pqm = dm.getMetadata();
- metadata.add(pqm.get(0)); // add first component (this is the actuator)
- // Skip the next component as this is the timestamp used to merge the data
- for(int i=2;i sqm = msCheck.getMetadata();
- // Skip first two components of the message as this is the timestamp
- for(int i=2;imilliseconds || (currMilliCheck==milliseconds && currNanoCheck>nanoOffset)){
- break;
- }
-
- DataMessage ms = (DataMessage) secondaryQueue.take();
-
- currData = ms.getData();
- // Remove timestamps
- currData.remove(0);
- currData.remove(0);
- }
- else{
- // No DataMessage
- logger.warning("No datamessage received!");
- break;
- }
- }
-
- if(currData!=null){
- try{
- // Add data to primary data queue message and put it into the out queue
- dm.getData().addAll(currData);
- dm.setMetadata(metadata);
- eventbus.post(dm);
- }
- catch(NullPointerException e){
- logger.log(Level.SEVERE, "Discarding message because of some missing data from the scr source", e);
- }
- }
- else{
- logger.warning("Discarding message because of missing data from the scr source");
- }
- }
-
-
- m = primaryQueue.take();
- }
-
- // Add the end of stream message of the primary queue
- eventbus.post(m);
-
- // Clear all remaining messages in secondary queue
- secondaryQueue.clear();
- }
-}
diff --git a/ch.psi.fda.xscan/src/test/java/ch/psi/fda/core/loops/cr/ParallelCrlogicStreamMergeTest.java b/ch.psi.fda.xscan/src/test/java/ch/psi/fda/core/loops/cr/ParallelCrlogicStreamMergeTest.java
index 380a467..018c15a 100644
--- a/ch.psi.fda.xscan/src/test/java/ch/psi/fda/core/loops/cr/ParallelCrlogicStreamMergeTest.java
+++ b/ch.psi.fda.xscan/src/test/java/ch/psi/fda/core/loops/cr/ParallelCrlogicStreamMergeTest.java
@@ -21,10 +21,11 @@ package ch.psi.fda.core.loops.cr;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
+import java.util.Queue;
+import java.util.concurrent.ArrayBlockingQueue;
import java.util.logging.Logger;
+import org.junit.Assert;
import org.junit.Test;
import com.google.common.eventbus.EventBus;
@@ -42,70 +43,89 @@ public class ParallelCrlogicStreamMergeTest {
@Test
public void testMerge() throws InterruptedException {
+ final Queue expectedValues = new ArrayBlockingQueue<>(3);
+ expectedValues.add(new double[]{0.0035, 1.0});
+ expectedValues.add(new double[]{0.015, 1.0});
+ expectedValues.add(new double[]{0.026, 3.0});
+
List dmm = new ArrayList<>();
dmm.add(new Metadata("cractuator"));
dmm.add(new Metadata("tstamp"));
- BlockingQueue dataQueue = new LinkedBlockingQueue();
- DataMessage dm = new DataMessage(dmm);
- dm.getData().add(0.0035d);
- dm.getData().add(10.000000123);
- dataQueue.add(dm);
+ EventBus dataQueue2 = new EventBus();
+ EventBus dataQueue = new EventBus();
- dm = new DataMessage(dmm);
- dm.getData().add(0.015);
- dm.getData().add(10.000000143);
- dataQueue.add(dm);
-
- dm = new DataMessage(dmm);
- dm.getData().add(0.026);
- dm.getData().add(10.000000163);
- dataQueue.add(dm);
-
- dataQueue.add(new EndOfStreamMessage());
-
-
- List dmm2 = new ArrayList<>();
- dmm2.add(new Metadata("milli"));
- dmm2.add(new Metadata("nano"));
- dmm2.add(new Metadata("sensor1"));
-
- BlockingQueue dataQueue2 = new LinkedBlockingQueue();
- DataMessage dm2 = new DataMessage(dmm2);
- dm2.getData().add(9000d);
- dm2.getData().add(122d);
- dm2.getData().add(0.1d);
- dataQueue2.add(dm2);
-
- dm2 = new DataMessage(dmm2);
- dm2.getData().add(10000d);
- dm2.getData().add(122d);
- dm2.getData().add(1d);
- dataQueue2.add(dm2);
-
- dm2 = new DataMessage(dmm2);
- dm2.getData().add(10000d);
- dm2.getData().add(153d);
- dm2.getData().add(2d);
- dataQueue2.add(dm2);
-
- dm2 = new DataMessage(dmm2);
- dm2.getData().add(10000d);
- dm2.getData().add(162d);
- dm2.getData().add(3d);
- dataQueue2.add(dm2);
-
- dataQueue2.add(new EndOfStreamMessage());
-
EventBus b = new EventBus();
b.register(new Object(){
@Subscribe
public void onMessage(Message m){
logger.info(m.toString());
+ if(m instanceof DataMessage){
+ DataMessage dm = (DataMessage) m;
+ double[] expected = expectedValues.remove();
+
+ Assert.assertEquals(expected[0], (double)dm.getData().get(0), 0.001);
+ Assert.assertEquals(expected[1], (double)dm.getData().get(1), 0.001);
+ }
}
});
- ParallelCrlogicStreamMerge streamMerge = new ParallelCrlogicStreamMerge(dataQueue, dataQueue2, b);
- streamMerge.merge();
+
+ MergeLogic streamMerge = new MergeLogic(dataQueue, dataQueue2, b);
+ streamMerge.enable();
+
+ // We need to fill the second queue first as otherwise the logic will fail
+ List dmm2 = new ArrayList<>();
+ dmm2.add(new Metadata("milli"));
+ dmm2.add(new Metadata("nano"));
+ dmm2.add(new Metadata("sensor1"));
+
+
+ DataMessage dm2 = new DataMessage(dmm2);
+ dm2.getData().add(9000d);
+ dm2.getData().add(122d);
+ dm2.getData().add(0.1d);
+ dataQueue2.post(dm2);
+
+ dm2 = new DataMessage(dmm2);
+ dm2.getData().add(10000d);
+ dm2.getData().add(122d);
+ dm2.getData().add(1d);
+ dataQueue2.post(dm2);
+
+ dm2 = new DataMessage(dmm2);
+ dm2.getData().add(10000d);
+ dm2.getData().add(153d);
+ dm2.getData().add(2d);
+ dataQueue2.post(dm2);
+
+ dm2 = new DataMessage(dmm2);
+ dm2.getData().add(10000d);
+ dm2.getData().add(162d);
+ dm2.getData().add(3d);
+ dataQueue2.post(dm2);
+
+ dataQueue2.post(new EndOfStreamMessage());
+
+
+ DataMessage dm = new DataMessage(dmm);
+ dm.getData().add(0.0035d);
+ dm.getData().add(10.000000123);
+ dataQueue.post(dm);
+
+ dm = new DataMessage(dmm);
+ dm.getData().add(0.015);
+ dm.getData().add(10.000000143);
+ dataQueue.post(dm);
+
+ dm = new DataMessage(dmm);
+ dm.getData().add(0.026);
+ dm.getData().add(10.000000163);
+ dataQueue.post(dm);
+
+ dataQueue.post(new EndOfStreamMessage());
+
+ streamMerge.disable();
+
}
/**
@@ -115,61 +135,9 @@ public class ParallelCrlogicStreamMergeTest {
@Test
public void testMergeSecondQueueLessElements() throws InterruptedException {
- List dmm = new ArrayList<>();
- dmm.add(new Metadata("cractuator"));
- dmm.add(new Metadata("tstamp"));
+ EventBus dataQueue = new EventBus();
+ EventBus dataQueue2 = new EventBus();
- BlockingQueue dataQueue = new LinkedBlockingQueue();
- DataMessage dm = new DataMessage(dmm);
- dm.getData().add(0.0035d);
- dm.getData().add(10.000000123);
- dataQueue.add(dm);
-
- dm = new DataMessage(dmm);
- dm.getData().add(0.015);
- dm.getData().add(10.000000143);
- dataQueue.add(dm);
-
-// dm = new DataMessage(dmm);
-// dm.getData().add(0.026);
-// dm.getData().add(10.000000163);
-// dataQueue.add(dm);
-
- dataQueue.add(new EndOfStreamMessage());
-
-
- List dmm2 = new ArrayList<>();
- dmm2.add(new Metadata("milli"));
- dmm2.add(new Metadata("nano"));
- dmm2.add(new Metadata("sensor1"));
-
- BlockingQueue dataQueue2 = new LinkedBlockingQueue();
- DataMessage dm2 = new DataMessage(dmm2);
-// dm2.getData().add(9000d);
-// dm2.getData().add(122d);
-// dm2.getData().add(0.1d);
-// dataQueue2.add(dm2);
-//
-// dm2 = new DataMessage(dmm2);
-// dm2.getData().add(10000d);
-// dm2.getData().add(122d);
-// dm2.getData().add(1d);
-// dataQueue2.add(dm2);
-
- dm2 = new DataMessage(dmm2);
- dm2.getData().add(10000d);
- dm2.getData().add(153d);
- dm2.getData().add(2d);
- dataQueue2.add(dm2);
-
-// dm2 = new DataMessage(dmm2);
-// dm2.getData().add(10000d);
-// dm2.getData().add(170d);
-// dm2.getData().add(3d);
-// dataQueue2.add(dm2);
-
- dataQueue2.add(new EndOfStreamMessage());
-
EventBus b = new EventBus();
b.register(new Object(){
@Subscribe
@@ -177,8 +145,68 @@ public class ParallelCrlogicStreamMergeTest {
logger.info(m.toString());
}
});
- ParallelCrlogicStreamMerge streamMerge = new ParallelCrlogicStreamMerge(dataQueue, dataQueue2, b);
- streamMerge.merge();
+ MergeLogic streamMerge = new MergeLogic(dataQueue, dataQueue2, b);
+
+ streamMerge.enable();
+
+
+ // We need to fill the second queue first as otherwise the logic will fail
+ List dmm2 = new ArrayList<>();
+ dmm2.add(new Metadata("milli"));
+ dmm2.add(new Metadata("nano"));
+ dmm2.add(new Metadata("sensor1"));
+
+ DataMessage dm2 = new DataMessage(dmm2);
+ dm2.getData().add(9000d);
+ dm2.getData().add(122d);
+ dm2.getData().add(0.1d);
+ dataQueue2.post(dm2);
+//
+// dm2 = new DataMessage(dmm2);
+// dm2.getData().add(10000d);
+// dm2.getData().add(122d);
+// dm2.getData().add(1d);
+// dataQueue2.add(dm2);
+
+// dm2 = new DataMessage(dmm2);
+// dm2.getData().add(10000d);
+// dm2.getData().add(153d);
+// dm2.getData().add(2d);
+// dataQueue2.post(dm2);
+
+// dm2 = new DataMessage(dmm2);
+// dm2.getData().add(10000d);
+// dm2.getData().add(170d);
+// dm2.getData().add(3d);
+// dataQueue2.add(dm2);
+
+ dataQueue2.post(new EndOfStreamMessage());
+
+
+
+ List dmm = new ArrayList<>();
+ dmm.add(new Metadata("cractuator"));
+ dmm.add(new Metadata("tstamp"));
+
+
+ DataMessage dm = new DataMessage(dmm);
+ dm.getData().add(0.0035d);
+ dm.getData().add(10.000000123);
+ dataQueue.post(dm);
+
+ dm = new DataMessage(dmm);
+ dm.getData().add(0.015);
+ dm.getData().add(10.000000143);
+ dataQueue.post(dm);
+
+// dm = new DataMessage(dmm);
+// dm.getData().add(0.026);
+// dm.getData().add(10.000000163);
+// dataQueue.add(dm);
+
+ dataQueue.post(new EndOfStreamMessage());
+
+ streamMerge.disable();
}
}