ITQC/gls_docs
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ATEST-81:

Browse Source 
- tracking down bugs and trying to fix
This commit is contained in:
Zellweger Christof Ralf
2015-07-01 16:51:22 +02:00
parent 2d7c299ef7
commit 6dfa14082d
12 changed files with 40 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/ch/psi/daq/rest/RestApplication.java
View File

@ -4,17 +4,12 @@ import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.boot.context.web.SpringBootServletInitializer;
import org.springframework.context.annotation.Import;
import ch.psi.daq.hazelcast.config.HazelcastClientConfig;
import ch.psi.daq.hazelcast.config.HazelcastConfig;
/**
* Entry point to our rest-frontend of the data acquisition (DAQ) application which most importantly
* wires all the @RestController annotated classes.
*/
@SpringBootApplication
@Import({HazelcastConfig.class, HazelcastClientConfig.class})
public class RestApplication extends SpringBootServletInitializer {

6
src/main/java/ch/psi/daq/rest/config/RestConfig.java
View File

@ -13,6 +13,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import ch.psi.daq.common.statistic.StorelessStatistics;
import ch.psi.daq.domain.cassandra.ChannelEvent;
import ch.psi.daq.hazelcast.config.HazelcastClientConfig;
import ch.psi.daq.hazelcast.config.HazelcastConfig;
import ch.psi.daq.rest.model.PropertyFilterMixin;
import ch.psi.daq.rest.response.ResponseStreamWriter;
@ -30,8 +31,9 @@ public class RestConfig {
// see:
// https://jira.spring.io/browse/SPR-10409?focusedCommentId=101393&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-101393
@Configuration
@Import({HazelcastConfig.class})
static class InnerConfiguration { }
@Import({HazelcastConfig.class, HazelcastClientConfig.class})
static class InnerConfiguration {
}
@Bean
public JsonFactory jsonFactory() {

6
src/main/java/ch/psi/daq/rest/controller/DaqRestController.java
View File

@ -26,10 +26,10 @@ import ch.psi.daq.cassandra.writer.CassandraWriter;
import ch.psi.daq.domain.DataType;
import ch.psi.daq.domain.cassandra.ChannelEvent;
import ch.psi.daq.domain.cassandra.DataEvent;
import ch.psi.daq.hazelcast.query.AbstractQuery;
import ch.psi.daq.hazelcast.query.PulseRangeQuery;
import ch.psi.daq.hazelcast.query.TimeRangeQuery;
import ch.psi.daq.hazelcast.query.processor.QueryProcessor;
import ch.psi.daq.rest.queries.AbstractQuery;
import ch.psi.daq.rest.queries.PulseRangeQuery;
import ch.psi.daq.rest.queries.TimeRangeQuery;
import ch.psi.daq.rest.response.ResponseStreamWriter;
@RestController

194
src/main/java/ch/psi/daq/rest/queries/AbstractQuery.java
View File

@ -1,194 +0,0 @@
package ch.psi.daq.rest.queries;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import ch.psi.daq.cassandra.reader.Ordering;
import ch.psi.daq.hazelcast.query.Aggregation;
import ch.psi.daq.hazelcast.query.AggregationType;
import ch.psi.daq.hazelcast.query.Query;
import ch.psi.daq.hazelcast.query.bin.BinningStrategy;
import ch.psi.daq.hazelcast.query.bin.BinningStrategyFactory;
import ch.psi.daq.hazelcast.query.range.QueryRange;
/**
*
* @author zellweger_c
*
*/
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include = JsonTypeInfo.As.PROPERTY,
property = "queryType")
@JsonSubTypes(
{
@Type(value = PulseRangeQuery.class, name = "pulserange"),
@Type(value = TimeRangeQuery.class, name = "timerange"),
})
public abstract class AbstractQuery implements Query {
private static final long serialVersionUID = 1L;
private static Logger logger = LoggerFactory.getLogger(AbstractQuery.class);
private List<String> channels;
private LinkedHashSet<String> fields;
private List<Aggregation> aggregations;
private AggregationType aggregationType;
private boolean aggregateChannels;
private Ordering ordering;
private BinningStrategy binningStrategy;
private BinningStrategyEnum binningStrategyEnum;
private QueryRange queryRange;
/**
* Constructor.
*
* @param ordering whether to add a 'orderBy' clause into the database query
* @param channels all the channelIds (channel names) we want to query
* @param fields the fields (who map to fields in the DB) we are interested in returning to the
* client, needs to be in insertion order (hence the {@link LinkedHashSet} type)
* @param binningStrategyEnum enum that maps the user's String to a concrete
* {@link BinningStrategy} implementation
* @param binLengthOrCount depending on the chosen binning strategy, this field defines either the
* count (how many pulse ids are to be put inside 1 bin) or the time frame for 1 bin
* @param aggregateChannels whether aggregation will include all channels, default is on a
* per-channel basis
* @param aggregationType defines whether aggregation takes place in an index- or value-based
* manner
* @param aggregations list of aggregations / statistics to calculate, e.g. min, max and average
* @param queryRange object containing the ranges for either pulse-based queries or time-based
* queries
*/
@JsonCreator
public AbstractQuery(
// note that those annotations are needed for the polymorphic
// mapping to work correctly
@JsonProperty(value = "ordering") Ordering ordering,
@JsonProperty(value = "channels") List<String> channels,
@JsonProperty(value = "fields") LinkedHashSet<String> fields,
@JsonProperty(value = "binningStrategy") BinningStrategyEnum binningStrategyEnum,
@JsonProperty(value = "binDuration") long binLengthOrCount,
@JsonProperty(value = "aggregateChannels") boolean aggregateChannels,
@JsonProperty(value = "aggregationType") AggregationType aggregationType,
@JsonProperty(value = "aggregations") List<Aggregation> aggregations,
@JsonProperty(value = "queryRange") QueryRange queryRange) {
this.ordering = ordering;
this.aggregateChannels = aggregateChannels;
this.aggregationType = aggregationType;
this.aggregations = aggregations;
this.queryRange = queryRange;
Assert.notNull(channels, "channel name must not be null.");
Assert.notNull(fields, "field, i.e. property, names must not be null.");
this.channels = channels;
this.fields = fields;
this.binningStrategyEnum = binningStrategyEnum; // can be null: default then will be
// BinCountBinningStrategy
if (binningStrategyEnum != null) {
switch (binningStrategyEnum) {
case count:
this.binningStrategy = BinningStrategyFactory.getBinningStrategy(getQueryRange(), (int) binLengthOrCount);
break;
case length:
this.binningStrategy = BinningStrategyFactory.getBinningStrategy(getQueryRange(), binLengthOrCount);
break;
default:
logger.warn("No binning strategy has been set. Selecting BinningStrategyBinCount.");
this.binningStrategy = BinningStrategyFactory.getBinningStrategy(getQueryRange(), (int) binLengthOrCount);
}
} else {
this.binningStrategy = BinningStrategyFactory.getBinningStrategy(getQueryRange(), (int) binLengthOrCount);
}
}
/**
* {@inheritDoc}
*/
@Override
public List<String> getChannels() {
return Collections.unmodifiableList(channels);
}
/**
* {@inheritDoc}
*/
@Override
public Ordering getOrdering() {
return ordering;
}
/**
* {@inheritDoc}
*/
@Override
public AggregationType getAggregationType() {
return aggregationType;
}
@Override
public boolean isAggregateChannels() {
return aggregateChannels;
}
public BinningStrategyEnum getBinningStrategyEnum() {
return binningStrategyEnum;
}
@Override
public BinningStrategy getBinningStrategy() {
return binningStrategy;
}
public LinkedHashSet<String> getFields() {
return fields;
}
public List<Aggregation> getAggregations() {
return aggregations;
}
/**
* {@inheritDoc}
*/
@Override
public QueryRange getQueryRange() {
return queryRange;
}
public void setQueryRange(QueryRange queryRange) {
this.queryRange = queryRange;
}
@Override
public String toString() {
return ReflectionToStringBuilder.reflectionToString(this);
}
}

16
src/main/java/ch/psi/daq/rest/queries/BinningStrategyEnum.java
View File

@ -1,16 +0,0 @@
package ch.psi.daq.rest.queries;
import ch.psi.daq.hazelcast.query.bin.BinningStrategy;
/**
* Defines the strategies that map to the similarly named {@link BinningStrategy} implementations.
* <p>
* This enum is used for type-safety and simplicity for the rest interface.
*
*/
public enum BinningStrategyEnum {
length,
count
}

71
src/main/java/ch/psi/daq/rest/queries/PulseRangeQuery.java
View File

@ -1,71 +0,0 @@
package ch.psi.daq.rest.queries;
import java.util.LinkedHashSet;
import java.util.List;
import org.apache.commons.lang3.builder.ToStringBuilder;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import ch.psi.daq.cassandra.reader.Ordering;
import ch.psi.daq.hazelcast.query.Aggregation;
import ch.psi.daq.hazelcast.query.AggregationType;
import ch.psi.daq.hazelcast.query.bin.BinningStrategy;
import ch.psi.daq.hazelcast.query.range.QueryRange;
/**
*
*/
public class PulseRangeQuery extends AbstractQuery {
private static final long serialVersionUID = 1L;
/**
* Constructor.
*
* @param ordering whether to add a 'orderBy' clause into the database query
* @param channels all the channelIds (channel names) we want to query
* @param fields the fields (who map to fields in the DB) we are interested in returning to the
* client, needs to be in insertion order (hence the {@link LinkedHashSet} type)
* @param binningStrategyEnum enum that maps the user's String to a concrete
* {@link BinningStrategy} implementation
* @param binLengthOrCount depending on the chosen binning strategy, this field defines either the
* count (how many pulse ids are to be put inside 1 bin) or the time frame for 1 bin
* @param aggregateChannels whether aggregation will include all channels, default is on a
* per-channel basis
* @param aggregationType defines whether aggregation takes place in an index- or value-based
* manner
* @param aggregations list of aggregations / statistics to calculate, e.g. min, max and average
* @param queryRange object containing the ranges for either pulse-based queries or time-based
* queries
*/
@JsonCreator
public PulseRangeQuery(
// note that those annotations are needed for the polymorphic
// mapping to work correctly
@JsonProperty(value = "ordering") Ordering ordering,
@JsonProperty(value = "channels") List<String> channels,
@JsonProperty(value = "fields") LinkedHashSet<String> fields,
@JsonProperty(value = "binningStrategy") BinningStrategyEnum binningStrategyEnum,
@JsonProperty(value = "binDuration") long binLengthOrCount,
@JsonProperty(value = "aggregateChannels") boolean aggregateChannels,
@JsonProperty(value = "aggregationType") AggregationType aggregationType,
@JsonProperty(value = "aggregations") List<Aggregation> aggregations,
@JsonProperty(value = "queryRange") QueryRange queryRange) {
super(ordering, channels, fields, binningStrategyEnum, binLengthOrCount, aggregateChannels, aggregationType,
aggregations, queryRange);
}
/**
* {@inheritDoc}
*/
@Override
public String toString() {
return ToStringBuilder.reflectionToString(this);
}
}

97
src/main/java/ch/psi/daq/rest/queries/TimeRangeQuery.java
View File

@ -1,97 +0,0 @@
package ch.psi.daq.rest.queries;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.LinkedHashSet;
import java.util.List;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import ch.psi.daq.cassandra.reader.Ordering;
import ch.psi.daq.hazelcast.query.Aggregation;
import ch.psi.daq.hazelcast.query.AggregationType;
import ch.psi.daq.hazelcast.query.bin.BinningStrategy;
import ch.psi.daq.hazelcast.query.range.QueryRange;
public class TimeRangeQuery extends AbstractQuery {
private static final long serialVersionUID = 1L;
public static final String DATE_FORMAT_STRING = "yyyy/MM/dd hh:mm:ss.SSS";
private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat(DATE_FORMAT_STRING);
private static Logger logger = LoggerFactory.getLogger(TimeRangeQuery.class);
/**
* Constructor
*
* @param ordering whether to add a 'orderBy' clause into the database query
* @param channels all the channelIds (channel names) we want to query
* @param fields the fields (who map to fields in the DB) we are interested in returning to the
* client, needs to be in insertion order (hence the {@link LinkedHashSet} type)
* @param binningStrategyEnum enum that maps the user's String to a concrete
* {@link BinningStrategy} implementation
* @param binLengthOrCount depending on the chosen binning strategy, this field defines either the
* count (how many pulse ids are to be put inside 1 bin) or the time frame for 1 bin
* @param aggregateChannels whether aggregation will include all channels, default is on a
* per-channel basis
* @param aggregationType defines whether aggregation takes place in an index- or value-based
* manner
* @param aggregations list of aggregations / statistics to calculate, e.g. min, max and average
* @param queryRange object containing the ranges for either pulse-based queries or time-based
* queries
* @param startDateTime if set, the date string (format is:
* {@link TimeRangeQuery#DATE_FORMAT_STRING} will be parsed and converted into
* milliseconds
* @param endDateTime set, the date string (format is: {@link TimeRangeQuery#DATE_FORMAT_STRING}
* will be parsed and converted into milliseconds
*/
@JsonCreator
public TimeRangeQuery(
// note that those annotations are needed for the polymorphic
// mapping to work correctly
@JsonProperty(value = "ordering") Ordering ordering,
@JsonProperty(value = "channels") List<String> channels,
@JsonProperty(value = "fields") LinkedHashSet<String> fields,
@JsonProperty(value = "binningStrategy") BinningStrategyEnum binningStrategyEnum,
@JsonProperty(value = "binDuration") long binLengthOrCount,
@JsonProperty(value = "aggregateChannels") boolean aggregateChannels,
@JsonProperty(value = "aggregationType") AggregationType aggregationType,
@JsonProperty(value = "aggregations") List<Aggregation> aggregations,
@JsonProperty(value = "queryRange") QueryRange queryRange,
@JsonProperty(value = "startDateTime") String startDateTime,
@JsonProperty(value = "endDateTime") String endDateTime) {
super(ordering, channels, fields, binningStrategyEnum, binLengthOrCount, aggregateChannels, aggregationType,
aggregations, queryRange);
if (startDateTime != null && endDateTime != null) {
logger.info("startDateTime and endDateTime specified. This takes precedence over the start / end fields.");
try {
Date startDate = DATE_FORMAT.parse(startDateTime);
Date endDate = DATE_FORMAT.parse(endDateTime);
getQueryRange().setTimeRange(startDate.getTime(), queryRange.getStartNanos(), endDate.getTime(),
queryRange.getEndNanos());
} catch (ParseException e) {
logger.error("Parsing the start- and/or endDate was unsuccessful. "
+ "The format must be '" + DATE_FORMAT_STRING + "'", e);
}
}
}
/**
* {@inheritDoc}
*/
@Override
public String toString() {
return ToStringBuilder.reflectionToString(this);
}
}

6
src/main/java/ch/psi/daq/rest/response/ResponseStreamWriter.java
View File

@ -14,9 +14,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import ch.psi.daq.domain.cassandra.DataEvent;
import ch.psi.daq.rest.queries.AbstractQuery;
import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
@ -25,6 +22,9 @@ import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
import ch.psi.daq.domain.cassandra.DataEvent;
import ch.psi.daq.hazelcast.query.AbstractQuery;
/**
* Takes a Java 8 stream and writes it to the output stream provided by the {@link ServletResponse}
* of the current request.

18
src/test/java/ch/psi/daq/test/rest/controller/DaqRestControllerTest.java
View File

@ -10,19 +10,19 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.result.MockMvcResultHandlers;
import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
import ch.psi.daq.cassandra.reader.Ordering;
import ch.psi.daq.hazelcast.query.AggregationType;
import ch.psi.daq.hazelcast.query.range.QueryRange;
import ch.psi.daq.hazelcast.query.range.QueryRangeImpl;
import ch.psi.daq.rest.queries.BinningStrategyEnum;
import ch.psi.daq.rest.queries.PulseRangeQuery;
import ch.psi.daq.rest.queries.TimeRangeQuery;
import ch.psi.daq.test.rest.AbstractDaqRestTest;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import ch.psi.daq.cassandra.reader.Ordering;
import ch.psi.daq.hazelcast.query.AggregationType;
import ch.psi.daq.hazelcast.query.BinningStrategyEnum;
import ch.psi.daq.hazelcast.query.PulseRangeQuery;
import ch.psi.daq.hazelcast.query.TimeRangeQuery;
import ch.psi.daq.hazelcast.query.range.QueryRange;
import ch.psi.daq.hazelcast.query.range.QueryRangeImpl;
import ch.psi.daq.test.rest.AbstractDaqRestTest;
/**
* Tests the {@link DaqController} implementation.
*/

10
src/test/java/ch/psi/daq/test/rest/queries/AbstractQueryTestTest.java
View File

@ -8,18 +8,18 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.BlockJUnit4ClassRunner;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import ch.psi.daq.cassandra.reader.Ordering;
import ch.psi.daq.hazelcast.query.AggregationType;
import ch.psi.daq.hazelcast.query.BinningStrategyEnum;
import ch.psi.daq.hazelcast.query.TimeRangeQuery;
import ch.psi.daq.hazelcast.query.bin.BinningStrategyBinCount;
import ch.psi.daq.hazelcast.query.bin.BinningStrategyLengthPulse;
import ch.psi.daq.hazelcast.query.bin.BinningStrategyLengthTime;
import ch.psi.daq.hazelcast.query.range.QueryRange;
import ch.psi.daq.hazelcast.query.range.QueryRangeImpl;
import ch.psi.daq.rest.queries.BinningStrategyEnum;
import ch.psi.daq.rest.queries.TimeRangeQuery;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@RunWith(BlockJUnit4ClassRunner.class)
public class AbstractQueryTestTest extends AbstractQueryTest {

14
src/test/java/ch/psi/daq/test/rest/queries/TimeRangeQueryTest.java
View File

@ -8,16 +8,16 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.BlockJUnit4ClassRunner;
import ch.psi.daq.cassandra.reader.Ordering;
import ch.psi.daq.hazelcast.query.AggregationType;
import ch.psi.daq.hazelcast.query.range.QueryRange;
import ch.psi.daq.hazelcast.query.range.QueryRangeImpl;
import ch.psi.daq.rest.queries.BinningStrategyEnum;
import ch.psi.daq.rest.queries.TimeRangeQuery;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import ch.psi.daq.cassandra.reader.Ordering;
import ch.psi.daq.hazelcast.query.AggregationType;
import ch.psi.daq.hazelcast.query.BinningStrategyEnum;
import ch.psi.daq.hazelcast.query.TimeRangeQuery;
import ch.psi.daq.hazelcast.query.range.QueryRange;
import ch.psi.daq.hazelcast.query.range.QueryRangeImpl;
@RunWith(BlockJUnit4ClassRunner.class)
public class TimeRangeQueryTest extends AbstractQueryTest {

6
src/test/java/ch/psi/daq/test/rest/query/DummyQueryProcessor.java
View File

@ -10,14 +10,14 @@ import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import ch.psi.daq.domain.cassandra.ChannelEvent;
import ch.psi.daq.domain.cassandra.DataEvent;
import ch.psi.daq.hazelcast.query.Query;
import ch.psi.daq.hazelcast.query.processor.QueryProcessor;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
/**
* @author zellweger_c
*