fixed scr channel merge

This commit is contained in:
ebner committed 2015-08-18 16:23:36 +02:00
1 parent 89d9dd4d5e
commit f2a6590efc
4 files changed
+350 -310

No files matched your search

@@ -0,0 +1,207 @@
/**
*
* Copyright 2010 Paul Scherrer Institute. All rights reserved.
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This code is distributed in the hope that it will be useful,
* but without any warranty; without even the implied warranty of
* merchantability or fitness for a particular purpose. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this code. If not, see <http://www.gnu.org/licenses/>.
*
*/
package ch.psi.fda.core.loops.cr;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.logging.Level;
import java.util.logging.Logger;
import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.Subscribe;
import ch.psi.fda.messages.DataMessage;
import ch.psi.fda.messages.EndOfStreamMessage;
import ch.psi.fda.messages.Message;
import ch.psi.fda.messages.Metadata;
/**
* Class to merge two data streams into one. The secondary queues data is added to the primary queues data.
* The resulting queue therefor will hold the same amount of elements as the primary queue does.
* The datagroup flag set in the end of stream message will be set according to the flag set in the primary queue.
*/
public class MergeLogic {
private static final Logger logger = Logger.getLogger(MergeLogic.class.getName());
private final EventBus primaryEventBus;
private final EventBus secondaryEventBus;
private final EventBus outputEventBus;
private List<Metadata> metadata;
// Handler of the secondary event bus. We need to keep the reference on this
// to be able to unregister ...
private Object handler;
boolean firstPrimary = true;
boolean firstSecondary = true;
private BlockingQueue<DataMessage> queue = new LinkedBlockingQueue<>();
private List<Object> currentData = null;
public MergeLogic(EventBus primaryEventBus, EventBus secondaryEventBus, EventBus outputEventBus){
this.primaryEventBus = primaryEventBus;
this.secondaryEventBus = secondaryEventBus;
this.outputEventBus = outputEventBus;
}
// Enables the merge logic
public void enable(){
firstPrimary = true;
firstSecondary = true;
handler = new Object(){
@Subscribe
public void onMessage(Message m){
// Only queue data messages
if(m instanceof DataMessage){
queue.add((DataMessage) m);
}
}
};
primaryEventBus.register(this);
secondaryEventBus.register(handler);
}
// Disables the merge logic
public void disable(){
primaryEventBus.unregister(this);
secondaryEventBus.unregister(handler);
}
/**
* This is the master merging logic
* @param m Data message from primary event bus
*/
@Subscribe
public void onMessage(Message rawMessage){
if(rawMessage instanceof EndOfStreamMessage){
outputEventBus.post(rawMessage);
// Clear queue of secondary event bus
queue.clear();
return;
}
else if(!(rawMessage instanceof DataMessage)){
logger.warning("Message type not supported - ignore");
return;
}
DataMessage message = (DataMessage) rawMessage;
// Hack to remove the synchronization timestamp from the primary event bus
if(firstPrimary){
firstPrimary = false;
metadata = new ArrayList<>();
List<Metadata> pqm = message.getMetadata();
metadata.add(pqm.get(0)); // add first component (this is the actuator)
// Skip the next component as this is the timestamp used to merge the data
for(int i=2;i<pqm.size();i++){
metadata.add(pqm.get(i));
}
}
// TODO Merge logic
// Get and remove merge timestamp from the data of the message
Double timestamp = (Double) message.getData().remove(1);
long milliseconds = (long) (timestamp*1000);
long nanoOffset = (long)((timestamp*1000-milliseconds)*1000000);
int count = 10;
while(true){ // TODO Wait only for a limited amount of time ...
// Assumption: the secondary Queue holds at least the data up to the timestamp of the message handled by this function
DataMessage msCheck = queue.peek();
// No data is available need to wait for some time
// If still no data is available, throw an exception
if(msCheck == null){
if(count==0){
if(currentData==null){
throw new RuntimeException("No data comming in from SCR channels");
}
else{
break;
}
}
// Need to wait until data comes in ... (this case is for the beginning only)
logger.info("No data available");
try {
Thread.sleep(100);
} catch (InterruptedException e) {
logger.log(Level.INFO, "", e);
}
count--;
continue;
}
// If we handle the first message of the secondary event bus, we copy its metadata
if(firstSecondary){
firstSecondary = false;
List<Metadata> sqm = msCheck.getMetadata();
// Skip first two components of the message as this is the timestamp
for(int i=2;i<sqm.size();i++){
metadata.add(sqm.get(i));
}
}
// Check whether timestamp of the next message is bigger than the timestamp of the
// message from the primary queue - if the timestamp is bigger do not take message out of the queue
long currMilliCheck = ((Double) msCheck.getData().get(0)).longValue();
long currNanoCheck = ((Double) msCheck.getData().get(1)).longValue();
if(currMilliCheck>milliseconds || (currMilliCheck==milliseconds && currNanoCheck>nanoOffset)){
break;
}
try {
DataMessage currentDataMessage = queue.take();
currentData = currentDataMessage.getData();
// Remove timestamps used for synchronization
currentData.remove(0);
currentData.remove(0);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
if(currentData == null){
throw new RuntimeException("No SCR data arrived");
}
// Add data to primary data queue message and put it into the out queue
message.getData().addAll(currentData);
message.setMetadata(metadata);
outputEventBus.post(message);
}
}
@@ -21,24 +21,20 @@ package ch.psi.fda.core.loops.cr;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.Subscribe;
import ch.psi.fda.core.Action;
import ch.psi.fda.core.ActionLoop;
import ch.psi.fda.messages.Message;
public class ParallelCrlogic implements ActionLoop {
@@ -62,7 +58,7 @@ public class ParallelCrlogic implements ActionLoop {
*/
private List<Action> postActions;
private ParallelCrlogicStreamMerge merger;
private MergeLogic mergeLogic;
private final EventBus eventbus;
@@ -89,23 +85,9 @@ public class ParallelCrlogic implements ActionLoop {
this.preActions = new ArrayList<Action>();
this.postActions = new ArrayList<Action>();
final BlockingQueue<Message> q1 = new LinkedBlockingQueue<>();
final BlockingQueue<Message> q2 = new LinkedBlockingQueue<>();
crlogic.getEventBus().register(new Object(){
@Subscribe
public void onMessage(Message m){
q1.add(m);
}
});
scrlogic.getEventBus().register(new Object(){
@Subscribe
public void onMessage(Message m){
q2.add(m);
}
});
this.merger = new ParallelCrlogicStreamMerge(q1, q2, eventbus);
this.mergeLogic = new MergeLogic(crlogic.getEventBus(), scrlogic.getEventBus(), eventbus);
}
@Override
@@ -115,6 +97,8 @@ public class ParallelCrlogic implements ActionLoop {
@Override
public void execute() throws InterruptedException {
mergeLogic.enable();
// Execute pre actions
for(Action action: preActions){
action.execute();
@@ -175,8 +159,6 @@ public class ParallelCrlogic implements ActionLoop {
}
}
merger.merge();
// Wait until all threads have finished
service.shutdown();
service.awaitTermination(1, TimeUnit.MINUTES);
@@ -187,6 +169,7 @@ public class ParallelCrlogic implements ActionLoop {
action.execute();
}
mergeLogic.disable();
}
@Override
@@ -1,178 +0,0 @@
/**
*
* Copyright 2010 Paul Scherrer Institute. All rights reserved.
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This code is distributed in the hope that it will be useful,
* but without any warranty; without even the implied warranty of
* merchantability or fitness for a particular purpose. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this code. If not, see <http://www.gnu.org/licenses/>.
*
*/
package ch.psi.fda.core.loops.cr;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.logging.Level;
import java.util.logging.Logger;
import com.google.common.eventbus.EventBus;
import ch.psi.fda.messages.DataMessage;
import ch.psi.fda.messages.EndOfStreamMessage;
import ch.psi.fda.messages.Message;
import ch.psi.fda.messages.Metadata;
/**
* Class to merge two data streams into one. The secondary queues data is added to the primary queues data.
* The resulting queue therefor will hold the same amount of elements as the primary queue does.
* The datagroup flag set in the end of stream message will be set according to the flag set in the primary queue.
*/
public class ParallelCrlogicStreamMerge {
private static final Logger logger = Logger.getLogger(ParallelCrlogicStreamMerge.class.getName());
private BlockingQueue<Message> primaryQueue;
private BlockingQueue<Message> secondaryQueue;
private final EventBus eventbus;
private List<Metadata> metadata;
/**
* @param pqueue Primary/master queue
* @param squeue Secondary queue
*/
public ParallelCrlogicStreamMerge(BlockingQueue<Message> pqueue, BlockingQueue<Message> squeue, EventBus ebus){
this.primaryQueue = pqueue;
this.secondaryQueue = squeue;
this.eventbus = ebus;
}
/**
* Merge the streams based on the timestamp
* @throws InterruptedException
*/
public void merge() throws InterruptedException{
metadata = new ArrayList<>();
boolean firstPrimary = true;
boolean firstSecondary = true;
// Actual data of the secondary queue
List<Object> currData = null;
// Take first element of the primary queue (wait until message is available)
Message m = primaryQueue.take();
while(! (m instanceof EndOfStreamMessage)){
if(m instanceof DataMessage){
DataMessage dm = (DataMessage) m;
if(firstPrimary){
firstPrimary = false;
List<Metadata> pqm = dm.getMetadata();
metadata.add(pqm.get(0)); // add first component (this is the actuator)
// Skip the next component as this is the timestamp used to merge the data
for(int i=2;i<pqm.size();i++){
metadata.add(pqm.get(i));
}
}
// Get and remove merge timestamp from the data of the message
Double timestamp = (Double) dm.getData().remove(1);
long milliseconds = (long) (timestamp*1000);
long nanoOffset = (long)((timestamp*1000-milliseconds)*1000000);
while(true){
// Assumption: the secondary Queue holds at least the data up to the
// timestamp of the primary queue
// TODO this assumption is probably not TRUE and need to be checked for !!!!
Message mess = secondaryQueue.peek();
if(mess instanceof EndOfStreamMessage){
break;
}
if(mess == null){
break;
}
if(mess instanceof DataMessage){
// Check whether timestamp of the next message is bigger than the timestamp of the
// message from the primary queue - if the timestamp is bigger do not take message out of the queue
DataMessage msCheck = (DataMessage) mess;
if(firstSecondary){
firstSecondary = false;
List<Metadata> sqm = msCheck.getMetadata();
// Skip first two components of the message as this is the timestamp
for(int i=2;i<sqm.size();i++){
metadata.add(sqm.get(i));
}
}
long currMilliCheck = ((Double) msCheck.getData().get(0)).longValue();
long currNanoCheck = ((Double) msCheck.getData().get(1)).longValue();
if(currMilliCheck>milliseconds || (currMilliCheck==milliseconds && currNanoCheck>nanoOffset)){
break;
}
DataMessage ms = (DataMessage) secondaryQueue.take();
currData = ms.getData();
// Remove timestamps
currData.remove(0);
currData.remove(0);
}
else{
// No DataMessage
logger.warning("No datamessage received!");
break;
}
}
if(currData!=null){
try{
// Add data to primary data queue message and put it into the out queue
dm.getData().addAll(currData);
dm.setMetadata(metadata);
eventbus.post(dm);
}
catch(NullPointerException e){
logger.log(Level.SEVERE, "Discarding message because of some missing data from the scr source", e);
}
}
else{
logger.warning("Discarding message because of missing data from the scr source");
}
}
m = primaryQueue.take();
}
// Add the end of stream message of the primary queue
eventbus.post(m);
// Clear all remaining messages in secondary queue
secondaryQueue.clear();
}
}
@@ -21,10 +21,11 @@ package ch.psi.fda.core.loops.cr;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.Queue;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.logging.Logger;
import org.junit.Assert;
import org.junit.Test;
import com.google.common.eventbus.EventBus;
@@ -42,70 +43,89 @@ public class ParallelCrlogicStreamMergeTest {
@Test
public void testMerge() throws InterruptedException {
final Queue<double[]> expectedValues = new ArrayBlockingQueue<>(3);
expectedValues.add(new double[]{0.0035, 1.0});
expectedValues.add(new double[]{0.015, 1.0});
expectedValues.add(new double[]{0.026, 3.0});
List<Metadata> dmm = new ArrayList<>();
dmm.add(new Metadata("cractuator"));
dmm.add(new Metadata("tstamp"));
BlockingQueue<Message> dataQueue = new LinkedBlockingQueue<Message>();
DataMessage dm = new DataMessage(dmm);
dm.getData().add(0.0035d);
dm.getData().add(10.000000123);
dataQueue.add(dm);
EventBus dataQueue2 = new EventBus();
EventBus dataQueue = new EventBus();
dm = new DataMessage(dmm);
dm.getData().add(0.015);
dm.getData().add(10.000000143);
dataQueue.add(dm);
dm = new DataMessage(dmm);
dm.getData().add(0.026);
dm.getData().add(10.000000163);
dataQueue.add(dm);
dataQueue.add(new EndOfStreamMessage());
List<Metadata> dmm2 = new ArrayList<>();
dmm2.add(new Metadata("milli"));
dmm2.add(new Metadata("nano"));
dmm2.add(new Metadata("sensor1"));
BlockingQueue<Message> dataQueue2 = new LinkedBlockingQueue<Message>();
DataMessage dm2 = new DataMessage(dmm2);
dm2.getData().add(9000d);
dm2.getData().add(122d);
dm2.getData().add(0.1d);
dataQueue2.add(dm2);
dm2 = new DataMessage(dmm2);
dm2.getData().add(10000d);
dm2.getData().add(122d);
dm2.getData().add(1d);
dataQueue2.add(dm2);
dm2 = new DataMessage(dmm2);
dm2.getData().add(10000d);
dm2.getData().add(153d);
dm2.getData().add(2d);
dataQueue2.add(dm2);
dm2 = new DataMessage(dmm2);
dm2.getData().add(10000d);
dm2.getData().add(162d);
dm2.getData().add(3d);
dataQueue2.add(dm2);
dataQueue2.add(new EndOfStreamMessage());
EventBus b = new EventBus();
b.register(new Object(){
@Subscribe
public void onMessage(Message m){
logger.info(m.toString());
if(m instanceof DataMessage){
DataMessage dm = (DataMessage) m;
double[] expected = expectedValues.remove();
Assert.assertEquals(expected[0], (double)dm.getData().get(0), 0.001);
Assert.assertEquals(expected[1], (double)dm.getData().get(1), 0.001);
}
}
});
ParallelCrlogicStreamMerge streamMerge = new ParallelCrlogicStreamMerge(dataQueue, dataQueue2, b);
streamMerge.merge();
MergeLogic streamMerge = new MergeLogic(dataQueue, dataQueue2, b);
streamMerge.enable();
// We need to fill the second queue first as otherwise the logic will fail
List<Metadata> dmm2 = new ArrayList<>();
dmm2.add(new Metadata("milli"));
dmm2.add(new Metadata("nano"));
dmm2.add(new Metadata("sensor1"));
DataMessage dm2 = new DataMessage(dmm2);
dm2.getData().add(9000d);
dm2.getData().add(122d);
dm2.getData().add(0.1d);
dataQueue2.post(dm2);
dm2 = new DataMessage(dmm2);
dm2.getData().add(10000d);
dm2.getData().add(122d);
dm2.getData().add(1d);
dataQueue2.post(dm2);
dm2 = new DataMessage(dmm2);
dm2.getData().add(10000d);
dm2.getData().add(153d);
dm2.getData().add(2d);
dataQueue2.post(dm2);
dm2 = new DataMessage(dmm2);
dm2.getData().add(10000d);
dm2.getData().add(162d);
dm2.getData().add(3d);
dataQueue2.post(dm2);
dataQueue2.post(new EndOfStreamMessage());
DataMessage dm = new DataMessage(dmm);
dm.getData().add(0.0035d);
dm.getData().add(10.000000123);
dataQueue.post(dm);
dm = new DataMessage(dmm);
dm.getData().add(0.015);
dm.getData().add(10.000000143);
dataQueue.post(dm);
dm = new DataMessage(dmm);
dm.getData().add(0.026);
dm.getData().add(10.000000163);
dataQueue.post(dm);
dataQueue.post(new EndOfStreamMessage());
streamMerge.disable();
}
/**
@@ -115,61 +135,9 @@ public class ParallelCrlogicStreamMergeTest {
@Test
public void testMergeSecondQueueLessElements() throws InterruptedException {
List<Metadata> dmm = new ArrayList<>();
dmm.add(new Metadata("cractuator"));
dmm.add(new Metadata("tstamp"));
EventBus dataQueue = new EventBus();
EventBus dataQueue2 = new EventBus();
BlockingQueue<Message> dataQueue = new LinkedBlockingQueue<Message>();
DataMessage dm = new DataMessage(dmm);
dm.getData().add(0.0035d);
dm.getData().add(10.000000123);
dataQueue.add(dm);
dm = new DataMessage(dmm);
dm.getData().add(0.015);
dm.getData().add(10.000000143);
dataQueue.add(dm);
// dm = new DataMessage(dmm);
// dm.getData().add(0.026);
// dm.getData().add(10.000000163);
// dataQueue.add(dm);
dataQueue.add(new EndOfStreamMessage());
List<Metadata> dmm2 = new ArrayList<>();
dmm2.add(new Metadata("milli"));
dmm2.add(new Metadata("nano"));
dmm2.add(new Metadata("sensor1"));
BlockingQueue<Message> dataQueue2 = new LinkedBlockingQueue<Message>();
DataMessage dm2 = new DataMessage(dmm2);
// dm2.getData().add(9000d);
// dm2.getData().add(122d);
// dm2.getData().add(0.1d);
// dataQueue2.add(dm2);
//
// dm2 = new DataMessage(dmm2);
// dm2.getData().add(10000d);
// dm2.getData().add(122d);
// dm2.getData().add(1d);
// dataQueue2.add(dm2);
dm2 = new DataMessage(dmm2);
dm2.getData().add(10000d);
dm2.getData().add(153d);
dm2.getData().add(2d);
dataQueue2.add(dm2);
// dm2 = new DataMessage(dmm2);
// dm2.getData().add(10000d);
// dm2.getData().add(170d);
// dm2.getData().add(3d);
// dataQueue2.add(dm2);
dataQueue2.add(new EndOfStreamMessage());
EventBus b = new EventBus();
b.register(new Object(){
@Subscribe
@@ -177,8 +145,68 @@ public class ParallelCrlogicStreamMergeTest {
logger.info(m.toString());
}
});
ParallelCrlogicStreamMerge streamMerge = new ParallelCrlogicStreamMerge(dataQueue, dataQueue2, b);
streamMerge.merge();
MergeLogic streamMerge = new MergeLogic(dataQueue, dataQueue2, b);
streamMerge.enable();
// We need to fill the second queue first as otherwise the logic will fail
List<Metadata> dmm2 = new ArrayList<>();
dmm2.add(new Metadata("milli"));
dmm2.add(new Metadata("nano"));
dmm2.add(new Metadata("sensor1"));
DataMessage dm2 = new DataMessage(dmm2);
dm2.getData().add(9000d);
dm2.getData().add(122d);
dm2.getData().add(0.1d);
dataQueue2.post(dm2);
//
// dm2 = new DataMessage(dmm2);
// dm2.getData().add(10000d);
// dm2.getData().add(122d);
// dm2.getData().add(1d);
// dataQueue2.add(dm2);
// dm2 = new DataMessage(dmm2);
// dm2.getData().add(10000d);
// dm2.getData().add(153d);
// dm2.getData().add(2d);
// dataQueue2.post(dm2);
// dm2 = new DataMessage(dmm2);
// dm2.getData().add(10000d);
// dm2.getData().add(170d);
// dm2.getData().add(3d);
// dataQueue2.add(dm2);
dataQueue2.post(new EndOfStreamMessage());
List<Metadata> dmm = new ArrayList<>();
dmm.add(new Metadata("cractuator"));
dmm.add(new Metadata("tstamp"));
DataMessage dm = new DataMessage(dmm);
dm.getData().add(0.0035d);
dm.getData().add(10.000000123);
dataQueue.post(dm);
dm = new DataMessage(dmm);
dm.getData().add(0.015);
dm.getData().add(10.000000143);
dataQueue.post(dm);
// dm = new DataMessage(dmm);
// dm.getData().add(0.026);
// dm.getData().add(10.000000163);
// dataQueue.add(dm);
dataQueue.post(new EndOfStreamMessage());
streamMerge.disable();
}
}