diff --git a/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/MergeLogic.java b/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/MergeLogic.java new file mode 100644 index 0000000..120b378 --- /dev/null +++ b/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/MergeLogic.java @@ -0,0 +1,207 @@ +/** + * + * Copyright 2010 Paul Scherrer Institute. All rights reserved. + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This code is distributed in the hope that it will be useful, + * but without any warranty; without even the implied warranty of + * merchantability or fitness for a particular purpose. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + * + */ + +package ch.psi.fda.core.loops.cr; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.Subscribe; + +import ch.psi.fda.messages.DataMessage; +import ch.psi.fda.messages.EndOfStreamMessage; +import ch.psi.fda.messages.Message; +import ch.psi.fda.messages.Metadata; + +/** + * Class to merge two data streams into one. The secondary queues data is added to the primary queues data. + * The resulting queue therefor will hold the same amount of elements as the primary queue does. + * The datagroup flag set in the end of stream message will be set according to the flag set in the primary queue. + */ +public class MergeLogic { + + private static final Logger logger = Logger.getLogger(MergeLogic.class.getName()); + + private final EventBus primaryEventBus; + private final EventBus secondaryEventBus; + private final EventBus outputEventBus; + + private List metadata; + + // Handler of the secondary event bus. We need to keep the reference on this + // to be able to unregister ... + private Object handler; + + + boolean firstPrimary = true; + boolean firstSecondary = true; + + private BlockingQueue queue = new LinkedBlockingQueue<>(); + private List currentData = null; + + + public MergeLogic(EventBus primaryEventBus, EventBus secondaryEventBus, EventBus outputEventBus){ + this.primaryEventBus = primaryEventBus; + this.secondaryEventBus = secondaryEventBus; + this.outputEventBus = outputEventBus; + } + + // Enables the merge logic + public void enable(){ + + firstPrimary = true; + firstSecondary = true; + + handler = new Object(){ + @Subscribe + public void onMessage(Message m){ + // Only queue data messages + if(m instanceof DataMessage){ + queue.add((DataMessage) m); + } + } + }; + + primaryEventBus.register(this); + secondaryEventBus.register(handler); + } + + // Disables the merge logic + public void disable(){ + primaryEventBus.unregister(this); + secondaryEventBus.unregister(handler); + } + + + /** + * This is the master merging logic + * @param m Data message from primary event bus + */ + @Subscribe + public void onMessage(Message rawMessage){ + if(rawMessage instanceof EndOfStreamMessage){ + outputEventBus.post(rawMessage); + + // Clear queue of secondary event bus + queue.clear(); + return; + } + else if(!(rawMessage instanceof DataMessage)){ + logger.warning("Message type not supported - ignore"); + return; + } + + DataMessage message = (DataMessage) rawMessage; + + // Hack to remove the synchronization timestamp from the primary event bus + if(firstPrimary){ + firstPrimary = false; + + metadata = new ArrayList<>(); + List pqm = message.getMetadata(); + metadata.add(pqm.get(0)); // add first component (this is the actuator) + // Skip the next component as this is the timestamp used to merge the data + for(int i=2;i sqm = msCheck.getMetadata(); + // Skip first two components of the message as this is the timestamp + for(int i=2;imilliseconds || (currMilliCheck==milliseconds && currNanoCheck>nanoOffset)){ + break; + } + + try { + DataMessage currentDataMessage = queue.take(); + currentData = currentDataMessage.getData(); + // Remove timestamps used for synchronization + currentData.remove(0); + currentData.remove(0); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + } + + if(currentData == null){ + throw new RuntimeException("No SCR data arrived"); + } + + // Add data to primary data queue message and put it into the out queue + message.getData().addAll(currentData); + message.setMetadata(metadata); + outputEventBus.post(message); + } +} diff --git a/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/ParallelCrlogic.java b/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/ParallelCrlogic.java index a3f2037..cf840c9 100644 --- a/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/ParallelCrlogic.java +++ b/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/ParallelCrlogic.java @@ -21,24 +21,20 @@ package ch.psi.fda.core.loops.cr; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import com.google.common.eventbus.EventBus; -import com.google.common.eventbus.Subscribe; import ch.psi.fda.core.Action; import ch.psi.fda.core.ActionLoop; -import ch.psi.fda.messages.Message; public class ParallelCrlogic implements ActionLoop { @@ -62,7 +58,7 @@ public class ParallelCrlogic implements ActionLoop { */ private List postActions; - private ParallelCrlogicStreamMerge merger; + private MergeLogic mergeLogic; private final EventBus eventbus; @@ -89,23 +85,9 @@ public class ParallelCrlogic implements ActionLoop { this.preActions = new ArrayList(); this.postActions = new ArrayList(); - final BlockingQueue q1 = new LinkedBlockingQueue<>(); - final BlockingQueue q2 = new LinkedBlockingQueue<>(); - crlogic.getEventBus().register(new Object(){ - @Subscribe - public void onMessage(Message m){ - q1.add(m); - } - }); - scrlogic.getEventBus().register(new Object(){ - @Subscribe - public void onMessage(Message m){ - q2.add(m); - } - }); - this.merger = new ParallelCrlogicStreamMerge(q1, q2, eventbus); + this.mergeLogic = new MergeLogic(crlogic.getEventBus(), scrlogic.getEventBus(), eventbus); } @Override @@ -115,6 +97,8 @@ public class ParallelCrlogic implements ActionLoop { @Override public void execute() throws InterruptedException { + mergeLogic.enable(); + // Execute pre actions for(Action action: preActions){ action.execute(); @@ -175,8 +159,6 @@ public class ParallelCrlogic implements ActionLoop { } } - merger.merge(); - // Wait until all threads have finished service.shutdown(); service.awaitTermination(1, TimeUnit.MINUTES); @@ -187,6 +169,7 @@ public class ParallelCrlogic implements ActionLoop { action.execute(); } + mergeLogic.disable(); } @Override diff --git a/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/ParallelCrlogicStreamMerge.java b/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/ParallelCrlogicStreamMerge.java deleted file mode 100644 index 9a5cc24..0000000 --- a/ch.psi.fda.xscan/src/main/java/ch/psi/fda/core/loops/cr/ParallelCrlogicStreamMerge.java +++ /dev/null @@ -1,178 +0,0 @@ -/** - * - * Copyright 2010 Paul Scherrer Institute. All rights reserved. - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This code is distributed in the hope that it will be useful, - * but without any warranty; without even the implied warranty of - * merchantability or fitness for a particular purpose. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this code. If not, see . - * - */ - -package ch.psi.fda.core.loops.cr; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.logging.Level; -import java.util.logging.Logger; - -import com.google.common.eventbus.EventBus; - -import ch.psi.fda.messages.DataMessage; -import ch.psi.fda.messages.EndOfStreamMessage; -import ch.psi.fda.messages.Message; -import ch.psi.fda.messages.Metadata; - -/** - * Class to merge two data streams into one. The secondary queues data is added to the primary queues data. - * The resulting queue therefor will hold the same amount of elements as the primary queue does. - * The datagroup flag set in the end of stream message will be set according to the flag set in the primary queue. - */ -public class ParallelCrlogicStreamMerge { - - private static final Logger logger = Logger.getLogger(ParallelCrlogicStreamMerge.class.getName()); - - private BlockingQueue primaryQueue; - private BlockingQueue secondaryQueue; - - private final EventBus eventbus; - - private List metadata; - - /** - * @param pqueue Primary/master queue - * @param squeue Secondary queue - */ - public ParallelCrlogicStreamMerge(BlockingQueue pqueue, BlockingQueue squeue, EventBus ebus){ - this.primaryQueue = pqueue; - this.secondaryQueue = squeue; - - this.eventbus = ebus; - } - - /** - * Merge the streams based on the timestamp - * @throws InterruptedException - */ - public void merge() throws InterruptedException{ - - metadata = new ArrayList<>(); - - boolean firstPrimary = true; - boolean firstSecondary = true; - - // Actual data of the secondary queue - List currData = null; - - // Take first element of the primary queue (wait until message is available) - Message m = primaryQueue.take(); - - while(! (m instanceof EndOfStreamMessage)){ - - if(m instanceof DataMessage){ - - DataMessage dm = (DataMessage) m; - - if(firstPrimary){ - firstPrimary = false; - List pqm = dm.getMetadata(); - metadata.add(pqm.get(0)); // add first component (this is the actuator) - // Skip the next component as this is the timestamp used to merge the data - for(int i=2;i sqm = msCheck.getMetadata(); - // Skip first two components of the message as this is the timestamp - for(int i=2;imilliseconds || (currMilliCheck==milliseconds && currNanoCheck>nanoOffset)){ - break; - } - - DataMessage ms = (DataMessage) secondaryQueue.take(); - - currData = ms.getData(); - // Remove timestamps - currData.remove(0); - currData.remove(0); - } - else{ - // No DataMessage - logger.warning("No datamessage received!"); - break; - } - } - - if(currData!=null){ - try{ - // Add data to primary data queue message and put it into the out queue - dm.getData().addAll(currData); - dm.setMetadata(metadata); - eventbus.post(dm); - } - catch(NullPointerException e){ - logger.log(Level.SEVERE, "Discarding message because of some missing data from the scr source", e); - } - } - else{ - logger.warning("Discarding message because of missing data from the scr source"); - } - } - - - m = primaryQueue.take(); - } - - // Add the end of stream message of the primary queue - eventbus.post(m); - - // Clear all remaining messages in secondary queue - secondaryQueue.clear(); - } -} diff --git a/ch.psi.fda.xscan/src/test/java/ch/psi/fda/core/loops/cr/ParallelCrlogicStreamMergeTest.java b/ch.psi.fda.xscan/src/test/java/ch/psi/fda/core/loops/cr/ParallelCrlogicStreamMergeTest.java index 380a467..018c15a 100644 --- a/ch.psi.fda.xscan/src/test/java/ch/psi/fda/core/loops/cr/ParallelCrlogicStreamMergeTest.java +++ b/ch.psi.fda.xscan/src/test/java/ch/psi/fda/core/loops/cr/ParallelCrlogicStreamMergeTest.java @@ -21,10 +21,11 @@ package ch.psi.fda.core.loops.cr; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.Queue; +import java.util.concurrent.ArrayBlockingQueue; import java.util.logging.Logger; +import org.junit.Assert; import org.junit.Test; import com.google.common.eventbus.EventBus; @@ -42,70 +43,89 @@ public class ParallelCrlogicStreamMergeTest { @Test public void testMerge() throws InterruptedException { + final Queue expectedValues = new ArrayBlockingQueue<>(3); + expectedValues.add(new double[]{0.0035, 1.0}); + expectedValues.add(new double[]{0.015, 1.0}); + expectedValues.add(new double[]{0.026, 3.0}); + List dmm = new ArrayList<>(); dmm.add(new Metadata("cractuator")); dmm.add(new Metadata("tstamp")); - BlockingQueue dataQueue = new LinkedBlockingQueue(); - DataMessage dm = new DataMessage(dmm); - dm.getData().add(0.0035d); - dm.getData().add(10.000000123); - dataQueue.add(dm); + EventBus dataQueue2 = new EventBus(); + EventBus dataQueue = new EventBus(); - dm = new DataMessage(dmm); - dm.getData().add(0.015); - dm.getData().add(10.000000143); - dataQueue.add(dm); - - dm = new DataMessage(dmm); - dm.getData().add(0.026); - dm.getData().add(10.000000163); - dataQueue.add(dm); - - dataQueue.add(new EndOfStreamMessage()); - - - List dmm2 = new ArrayList<>(); - dmm2.add(new Metadata("milli")); - dmm2.add(new Metadata("nano")); - dmm2.add(new Metadata("sensor1")); - - BlockingQueue dataQueue2 = new LinkedBlockingQueue(); - DataMessage dm2 = new DataMessage(dmm2); - dm2.getData().add(9000d); - dm2.getData().add(122d); - dm2.getData().add(0.1d); - dataQueue2.add(dm2); - - dm2 = new DataMessage(dmm2); - dm2.getData().add(10000d); - dm2.getData().add(122d); - dm2.getData().add(1d); - dataQueue2.add(dm2); - - dm2 = new DataMessage(dmm2); - dm2.getData().add(10000d); - dm2.getData().add(153d); - dm2.getData().add(2d); - dataQueue2.add(dm2); - - dm2 = new DataMessage(dmm2); - dm2.getData().add(10000d); - dm2.getData().add(162d); - dm2.getData().add(3d); - dataQueue2.add(dm2); - - dataQueue2.add(new EndOfStreamMessage()); - EventBus b = new EventBus(); b.register(new Object(){ @Subscribe public void onMessage(Message m){ logger.info(m.toString()); + if(m instanceof DataMessage){ + DataMessage dm = (DataMessage) m; + double[] expected = expectedValues.remove(); + + Assert.assertEquals(expected[0], (double)dm.getData().get(0), 0.001); + Assert.assertEquals(expected[1], (double)dm.getData().get(1), 0.001); + } } }); - ParallelCrlogicStreamMerge streamMerge = new ParallelCrlogicStreamMerge(dataQueue, dataQueue2, b); - streamMerge.merge(); + + MergeLogic streamMerge = new MergeLogic(dataQueue, dataQueue2, b); + streamMerge.enable(); + + // We need to fill the second queue first as otherwise the logic will fail + List dmm2 = new ArrayList<>(); + dmm2.add(new Metadata("milli")); + dmm2.add(new Metadata("nano")); + dmm2.add(new Metadata("sensor1")); + + + DataMessage dm2 = new DataMessage(dmm2); + dm2.getData().add(9000d); + dm2.getData().add(122d); + dm2.getData().add(0.1d); + dataQueue2.post(dm2); + + dm2 = new DataMessage(dmm2); + dm2.getData().add(10000d); + dm2.getData().add(122d); + dm2.getData().add(1d); + dataQueue2.post(dm2); + + dm2 = new DataMessage(dmm2); + dm2.getData().add(10000d); + dm2.getData().add(153d); + dm2.getData().add(2d); + dataQueue2.post(dm2); + + dm2 = new DataMessage(dmm2); + dm2.getData().add(10000d); + dm2.getData().add(162d); + dm2.getData().add(3d); + dataQueue2.post(dm2); + + dataQueue2.post(new EndOfStreamMessage()); + + + DataMessage dm = new DataMessage(dmm); + dm.getData().add(0.0035d); + dm.getData().add(10.000000123); + dataQueue.post(dm); + + dm = new DataMessage(dmm); + dm.getData().add(0.015); + dm.getData().add(10.000000143); + dataQueue.post(dm); + + dm = new DataMessage(dmm); + dm.getData().add(0.026); + dm.getData().add(10.000000163); + dataQueue.post(dm); + + dataQueue.post(new EndOfStreamMessage()); + + streamMerge.disable(); + } /** @@ -115,61 +135,9 @@ public class ParallelCrlogicStreamMergeTest { @Test public void testMergeSecondQueueLessElements() throws InterruptedException { - List dmm = new ArrayList<>(); - dmm.add(new Metadata("cractuator")); - dmm.add(new Metadata("tstamp")); + EventBus dataQueue = new EventBus(); + EventBus dataQueue2 = new EventBus(); - BlockingQueue dataQueue = new LinkedBlockingQueue(); - DataMessage dm = new DataMessage(dmm); - dm.getData().add(0.0035d); - dm.getData().add(10.000000123); - dataQueue.add(dm); - - dm = new DataMessage(dmm); - dm.getData().add(0.015); - dm.getData().add(10.000000143); - dataQueue.add(dm); - -// dm = new DataMessage(dmm); -// dm.getData().add(0.026); -// dm.getData().add(10.000000163); -// dataQueue.add(dm); - - dataQueue.add(new EndOfStreamMessage()); - - - List dmm2 = new ArrayList<>(); - dmm2.add(new Metadata("milli")); - dmm2.add(new Metadata("nano")); - dmm2.add(new Metadata("sensor1")); - - BlockingQueue dataQueue2 = new LinkedBlockingQueue(); - DataMessage dm2 = new DataMessage(dmm2); -// dm2.getData().add(9000d); -// dm2.getData().add(122d); -// dm2.getData().add(0.1d); -// dataQueue2.add(dm2); -// -// dm2 = new DataMessage(dmm2); -// dm2.getData().add(10000d); -// dm2.getData().add(122d); -// dm2.getData().add(1d); -// dataQueue2.add(dm2); - - dm2 = new DataMessage(dmm2); - dm2.getData().add(10000d); - dm2.getData().add(153d); - dm2.getData().add(2d); - dataQueue2.add(dm2); - -// dm2 = new DataMessage(dmm2); -// dm2.getData().add(10000d); -// dm2.getData().add(170d); -// dm2.getData().add(3d); -// dataQueue2.add(dm2); - - dataQueue2.add(new EndOfStreamMessage()); - EventBus b = new EventBus(); b.register(new Object(){ @Subscribe @@ -177,8 +145,68 @@ public class ParallelCrlogicStreamMergeTest { logger.info(m.toString()); } }); - ParallelCrlogicStreamMerge streamMerge = new ParallelCrlogicStreamMerge(dataQueue, dataQueue2, b); - streamMerge.merge(); + MergeLogic streamMerge = new MergeLogic(dataQueue, dataQueue2, b); + + streamMerge.enable(); + + + // We need to fill the second queue first as otherwise the logic will fail + List dmm2 = new ArrayList<>(); + dmm2.add(new Metadata("milli")); + dmm2.add(new Metadata("nano")); + dmm2.add(new Metadata("sensor1")); + + DataMessage dm2 = new DataMessage(dmm2); + dm2.getData().add(9000d); + dm2.getData().add(122d); + dm2.getData().add(0.1d); + dataQueue2.post(dm2); +// +// dm2 = new DataMessage(dmm2); +// dm2.getData().add(10000d); +// dm2.getData().add(122d); +// dm2.getData().add(1d); +// dataQueue2.add(dm2); + +// dm2 = new DataMessage(dmm2); +// dm2.getData().add(10000d); +// dm2.getData().add(153d); +// dm2.getData().add(2d); +// dataQueue2.post(dm2); + +// dm2 = new DataMessage(dmm2); +// dm2.getData().add(10000d); +// dm2.getData().add(170d); +// dm2.getData().add(3d); +// dataQueue2.add(dm2); + + dataQueue2.post(new EndOfStreamMessage()); + + + + List dmm = new ArrayList<>(); + dmm.add(new Metadata("cractuator")); + dmm.add(new Metadata("tstamp")); + + + DataMessage dm = new DataMessage(dmm); + dm.getData().add(0.0035d); + dm.getData().add(10.000000123); + dataQueue.post(dm); + + dm = new DataMessage(dmm); + dm.getData().add(0.015); + dm.getData().add(10.000000143); + dataQueue.post(dm); + +// dm = new DataMessage(dmm); +// dm.getData().add(0.026); +// dm.getData().add(10.000000163); +// dataQueue.add(dm); + + dataQueue.post(new EndOfStreamMessage()); + + streamMerge.disable(); } }