diff --git a/build.gradle b/build.gradle index e7c9c6b..346adb0 100644 --- a/build.gradle +++ b/build.gradle @@ -1,29 +1,21 @@ version = '1.0.0' buildscript { - dependencies { - classpath("org.springframework.boot:spring-boot-gradle-plugin:1.2.4.RELEASE") - } - repositories { - jcenter() - } + dependencies { classpath(libraries.spring_boot_gradle_plugin) } + repositories { jcenter() } } apply plugin: 'spring-boot' -repositories { - jcenter() -} +repositories { jcenter() } -springBoot { - // when using spring loaded turn on noverify - noverify = true -} +springBoot { // when using spring loaded turn on noverify + noverify = true } applicationDefaultJvmArgs = [ - "-Dfile.encoding=UTF-8", - // if you need to debug java agents: - "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006" + "-Dfile.encoding=UTF-8", + // if you need to debug java agents: + "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006" ] //configurations { @@ -31,24 +23,21 @@ applicationDefaultJvmArgs = [ //} dependencies { - compile (project(':ch.psi.daq.hazelcast')) - compile 'org.springframework.boot:spring-boot-starter-web:1.2.4.RELEASE' - compile 'com.google.code.gson:gson:2+' - compile 'org.apache.commons:commons-lang3:3.4' - - testCompile 'org.springframework.boot:spring-boot-starter-test:1.2.4.RELEASE' - testCompile 'org.skyscreamer:jsonassert:1.2.3' - testCompile 'com.jayway.jsonpath:json-path:2.0.0' + compile (project(':ch.psi.daq.hazelcast')) + compile libraries.spring_boot_starter_web + compile libraries.commons_lang + + testCompile libraries.spring_boot_starter_test + testCompile libraries.jsonassert + testCompile libraries.jsonpath } uploadArchives { - repositories { - mavenDeployer { - pom.artifactId = 'rest' - } - } + repositories { + mavenDeployer { pom.artifactId = 'rest' } + } } task dropItQueryREST(dependsOn: build) << { - exec{ executable "curl"; args "-X", "POST", "-F", "file=@build/libs/ch.psi.daq.rest-" + version + ".jar", "http://dropit.psi.ch:8080"; } + exec{ executable "curl"; args "-X", "POST", "-F", "file=@build/libs/ch.psi.daq.rest-" + version + ".jar", "http://dropit.psi.ch:8080"; } } \ No newline at end of file diff --git a/src/main/java/ch/psi/daq/rest/RestApplication.java b/src/main/java/ch/psi/daq/rest/RestApplication.java index 86a2c04..f61fce4 100644 --- a/src/main/java/ch/psi/daq/rest/RestApplication.java +++ b/src/main/java/ch/psi/daq/rest/RestApplication.java @@ -4,39 +4,12 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.boot.context.web.SpringBootServletInitializer; -import org.springframework.context.annotation.ComponentScan; /** - * Entry point to our rest-frontend of the Swissfel application which most importantly wires all the @RestController - * annotated classes. - *

- * - * This acts as a @Configuration class for Spring. As such it has @ComponentScan annotation that - * enables scanning for another Spring components in current package and its subpackages. - *

- * Another annotation is @EnableAutoConfiguration which tells Spring Boot to run autoconfiguration. - *

- * It also extends SpringBootServletInitializer which will configure Spring servlet for us, and - * overrides the configure() method to point to itself, so Spring can find the main configuration. - *

- * Finally, the main() method consists of single static call to SpringApplication.run(). - *

- * Methods annotated with @Bean are Java beans that are container-managed, i.e. managed by Spring. - * Whenever there are @Autowire, @Inject or similar annotations found in the code (which is being - * scanned through the @ComponentScan annotation), the container then knows how to create those - * beans and inject them accordingly. + * Entry point to our rest-frontend of the data acquisition (DAQ) application which most importantly + * wires all the @RestController annotated classes. */ @SpringBootApplication -// @Import(CassandraConfig.class) // either define the context to be imported, or see ComponentScan -// comment below -@ComponentScan(basePackages = { - "ch.psi.daq.cassandra.config", // define the package name with the CassandraConfig - // configuration, or @Import it (see above) - "ch.psi.daq.cassandra.reader", - "ch.psi.daq.cassandra.writer", - "ch.psi.daq.hazelcast", - "ch.psi.daq.rest", -}) public class RestApplication extends SpringBootServletInitializer { diff --git a/src/main/java/ch/psi/daq/rest/config/RestConfiguration.java b/src/main/java/ch/psi/daq/rest/config/RestConfig.java similarity index 50% rename from src/main/java/ch/psi/daq/rest/config/RestConfiguration.java rename to src/main/java/ch/psi/daq/rest/config/RestConfig.java index d904006..7b30cdf 100644 --- a/src/main/java/ch/psi/daq/rest/config/RestConfiguration.java +++ b/src/main/java/ch/psi/daq/rest/config/RestConfig.java @@ -2,12 +2,12 @@ package ch.psi.daq.rest.config; import java.util.Arrays; import java.util.List; +import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; import org.springframework.context.annotation.PropertySource; import org.springframework.core.env.Environment; import org.springframework.util.StringUtils; @@ -15,70 +15,41 @@ import org.springframework.util.StringUtils; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.ObjectMapper; -import com.hazelcast.client.HazelcastClient; -import com.hazelcast.client.config.ClientConfig; -import com.hazelcast.core.HazelcastInstance; -import com.hazelcast.core.ManagedContext; -import com.hazelcast.spring.context.SpringManagedContext; +import com.google.common.collect.Sets; import ch.psi.daq.common.statistic.StorelessStatistics; import ch.psi.daq.domain.cassandra.ChannelEvent; +import ch.psi.daq.hazelcast.config.HazelcastClientConfig; import ch.psi.daq.hazelcast.config.HazelcastConfig; -import ch.psi.daq.hazelcast.query.processor.QueryProcessor; -import ch.psi.daq.hazelcast.query.processor.cassandra.CassandraQueryProcessorDistributed; -import ch.psi.daq.rest.ResponseStreamWriter; import ch.psi.daq.rest.model.PropertyFilterMixin; +import ch.psi.daq.rest.response.ResponseStreamWriter; @Configuration -@PropertySource(value = { "classpath:rest.properties" }) -@PropertySource(value = { "file:${user.home}/.config/daq/rest.properties" }, ignoreResourceNotFound = true) -public class RestConfiguration { +@PropertySource(value = {"classpath:rest.properties"}) +@PropertySource(value = {"file:${user.home}/.config/daq/rest.properties"}, ignoreResourceNotFound = true) +public class RestConfig { - private static final Logger logger = LoggerFactory.getLogger(RestConfiguration.class); - @Autowired private Environment env; - - @Bean - public QueryProcessor queryProcessor() { - return new CassandraQueryProcessorDistributed(); + + // a nested configuration + // this guarantees that the ordering of the properties file is as expected + // see: + // https://jira.spring.io/browse/SPR-10409?focusedCommentId=101393&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-101393 + @Configuration + @Import({HazelcastConfig.class, HazelcastClientConfig.class}) + static class InnerConfiguration { } @Bean public JsonFactory jsonFactory() { return new JsonFactory(); } - + @Bean public ResponseStreamWriter responseStreamWriter() { return new ResponseStreamWriter(); } - - @Bean(name = HazelcastConfig.BEAN_NAME_HAZELCAST_CLIENT) - public HazelcastInstance hazelcastClientInstance() { - return HazelcastClient.newHazelcastClient(hazelcastClientConfig()); - } - - private ClientConfig hazelcastClientConfig() { - ClientConfig config = new ClientConfig(); - config.setManagedContext(managedContext()); - - config.getNetworkConfig().setAddresses(hazelcastMembers()); - - return config; - } - - - private ManagedContext managedContext() { - return new SpringManagedContext(); - } - - @Bean - public List hazelcastMembers() { - List clusterMembers = Arrays.asList(StringUtils.commaDelimitedListToStringArray(env.getProperty("hazelcast.initialcandidates"))); - logger.info("The following hosts have been defined to form a Hazelcast cluster: {}", clusterMembers); - return clusterMembers; - } @Bean public ObjectMapper objectMapper() { @@ -91,12 +62,14 @@ public class RestConfiguration { mapper.addMixIn(StorelessStatistics.class, PropertyFilterMixin.class); return mapper; } - - // a nested configuration - // this guarantees that the ordering of the properties file is as expected - // see: - // https://jira.spring.io/browse/SPR-10409?focusedCommentId=101393&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-101393 - // @Configuration - // @Import(CassandraConfig.class) - // static class InnerConfiguration { } + + @Bean + private Set defaultResponseFields() { + List defaultFields = Arrays.asList( + StringUtils.commaDelimitedListToStringArray( + env.getProperty("rest.default.response.fields") + )); + Set defaultResponseFields = Sets.newHashSet(defaultFields.iterator()); + return defaultResponseFields; + } } diff --git a/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java b/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java index 99d34c6..8b537e8 100644 --- a/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java +++ b/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java @@ -1,9 +1,15 @@ package ch.psi.daq.rest.controller; import java.io.IOException; +import java.util.Arrays; +import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.LongFunction; +import java.util.function.LongUnaryOperator; +import java.util.stream.Collectors; +import java.util.stream.LongStream; import java.util.stream.Stream; import javax.servlet.http.HttpServletResponse; @@ -11,18 +17,20 @@ import javax.servlet.http.HttpServletResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.Assert; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import ch.psi.daq.cassandra.writer.CassandraWriter; +import ch.psi.daq.domain.DataType; import ch.psi.daq.domain.cassandra.ChannelEvent; import ch.psi.daq.domain.cassandra.DataEvent; +import ch.psi.daq.hazelcast.query.AbstractQuery; +import ch.psi.daq.hazelcast.query.PulseRangeQuery; +import ch.psi.daq.hazelcast.query.TimeRangeQuery; import ch.psi.daq.hazelcast.query.processor.QueryProcessor; -import ch.psi.daq.rest.ResponseStreamWriter; -import ch.psi.daq.rest.queries.AbstractQuery; -import ch.psi.daq.rest.queries.PulseRangeQuery; -import ch.psi.daq.rest.queries.TimeRangeQuery; +import ch.psi.daq.rest.response.ResponseStreamWriter; @RestController public class DaqRestController { @@ -38,7 +46,6 @@ public class DaqRestController { @Autowired private QueryProcessor queryProcessor; - @RequestMapping(value = "/pulserange") public void pulseRange(@RequestBody PulseRangeQuery query, HttpServletResponse res) throws IOException { @@ -71,20 +78,68 @@ public class DaqRestController { responseStreamWriter.respond(flatStreams, query, res); } - + // ========================================================================================== + // TODO this is simply a method for initial / rudimentary testing - remove once further evolved @RequestMapping(value = "/write") - public void writeDummyEntry() { - int pulseId = 100; - ChannelEvent event = new ChannelEvent( - "dummy-test", - pulseId, - 0, - pulseId, - pulseId, - 0, - "data_" + UUID.randomUUID().toString()); + public long writeDummyEntry() { - CompletableFuture future = cassandraWriter.writeAsync(3, 0, event); + long startIndex = System.currentTimeMillis(); + + writeData(3,startIndex, 100, new String[]{"channel1", "channel2"}); + + return startIndex; + } + + private void writeData(int dataReplication, long startIndex, long nrOfElements, + String... channelNames) { + writeData(dataReplication, startIndex, nrOfElements, + i -> 2 * i, + i -> 2 * i, + i -> 2 * i, + i -> 2 * i, + i -> 2 * i, + i -> Long.valueOf(2 * i), + channelNames); + } + + private void writeData(int dataReplication, long startIndex, long nrOfElements, + LongUnaryOperator iocMillis, LongUnaryOperator iocNanos, LongUnaryOperator pulseIds, + LongUnaryOperator globalMillis, LongUnaryOperator globalNanos, LongFunction valueFunction, + String... channelNames) { + + Assert.notNull(channelNames); + + CompletableFuture future; + + List events = + Arrays.stream(channelNames) + .parallel() + .flatMap( + channelName -> { + Stream stream = + LongStream + .range(startIndex, startIndex + nrOfElements) + .parallel() + .mapToObj( + i -> { + Object value = valueFunction.apply(i); + return new ChannelEvent(channelName, + iocMillis.applyAsLong(i), + iocNanos.applyAsLong(i), + pulseIds.applyAsLong(i), + globalMillis.applyAsLong(i), + globalNanos.applyAsLong(i), + value, + DataType.getTypeName(value.getClass()) + ); + } + ); + + return stream; + }) + .collect(Collectors.toList()); + + future = cassandraWriter.writeAsync(dataReplication, (int) TimeUnit.HOURS.toSeconds(1), events); future.join(); } } diff --git a/src/main/java/ch/psi/daq/rest/queries/AbstractQuery.java b/src/main/java/ch/psi/daq/rest/queries/AbstractQuery.java deleted file mode 100644 index bcf67c3..0000000 --- a/src/main/java/ch/psi/daq/rest/queries/AbstractQuery.java +++ /dev/null @@ -1,192 +0,0 @@ -package ch.psi.daq.rest.queries; - -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; - -import org.apache.commons.lang3.builder.ReflectionToStringBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.Assert; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonSubTypes.Type; -import com.fasterxml.jackson.annotation.JsonTypeInfo; - -import ch.psi.daq.cassandra.reader.Ordering; -import ch.psi.daq.hazelcast.query.Aggregation; -import ch.psi.daq.hazelcast.query.AggregationType; -import ch.psi.daq.hazelcast.query.Query; -import ch.psi.daq.hazelcast.query.bin.BinningStrategy; -import ch.psi.daq.hazelcast.query.bin.BinningStrategyFactory; -import ch.psi.daq.hazelcast.query.range.QueryRange; - -/** - * - * @author zellweger_c - * - */ -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "queryType") -@JsonSubTypes( -{ - @Type(value = PulseRangeQuery.class, name = "pulserange"), - @Type(value = TimeRangeQuery.class, name = "timerange"), -}) -public abstract class AbstractQuery implements Query { - - private static Logger logger = LoggerFactory.getLogger(AbstractQuery.class); - - private List channels; - - private LinkedHashSet fields; - - private List aggregations; - - private AggregationType aggregationType; - - private boolean aggregateChannels; - - private Ordering ordering; - - private BinningStrategy binningStrategy; - - private BinningStrategyEnum binningStrategyEnum; - - private QueryRange queryRange; - - /** - * Constructor. - * - * @param ordering whether to add a 'orderBy' clause into the database query - * @param channels all the channelIds (channel names) we want to query - * @param fields the fields (who map to fields in the DB) we are interested in returning to the - * client, needs to be in insertion order (hence the {@link LinkedHashSet} type) - * @param binningStrategyEnum enum that maps the user's String to a concrete - * {@link BinningStrategy} implementation - * @param binLengthOrCount depending on the chosen binning strategy, this field defines either the - * count (how many pulse ids are to be put inside 1 bin) or the time frame for 1 bin - * @param aggregateChannels whether aggregation will include all channels, default is on a - * per-channel basis - * @param aggregationType defines whether aggregation takes place in an index- or value-based - * manner - * @param aggregations list of aggregations / statistics to calculate, e.g. min, max and average - * @param queryRange object containing the ranges for either pulse-based queries or time-based - * queries - */ - @JsonCreator - public AbstractQuery( - // note that those annotations are needed for the polymorphic - // mapping to work correctly - @JsonProperty(value = "ordering") Ordering ordering, - @JsonProperty(value = "channels") List channels, - @JsonProperty(value = "fields") LinkedHashSet fields, - @JsonProperty(value = "binningStrategy") BinningStrategyEnum binningStrategyEnum, - @JsonProperty(value = "binDuration") long binLengthOrCount, - @JsonProperty(value = "aggregateChannels") boolean aggregateChannels, - @JsonProperty(value = "aggregationType") AggregationType aggregationType, - @JsonProperty(value = "aggregations") List aggregations, - @JsonProperty(value = "queryRange") QueryRange queryRange) { - - this.ordering = ordering; - this.aggregateChannels = aggregateChannels; - this.aggregationType = aggregationType; - this.aggregations = aggregations; - this.queryRange = queryRange; - - Assert.notNull(channels, "channel name must not be null."); - Assert.notNull(fields, "field, i.e. property, names must not be null."); - - this.channels = channels; - this.fields = fields; - - this.binningStrategyEnum = binningStrategyEnum; // can be null: default then will be - // BinCountBinningStrategy - - - if (binningStrategyEnum != null) { - switch (binningStrategyEnum) { - case count: - this.binningStrategy = BinningStrategyFactory.getBinningStrategy(getQueryRange(), (int) binLengthOrCount); - break; - case length: - this.binningStrategy = BinningStrategyFactory.getBinningStrategy(getQueryRange(), binLengthOrCount); - break; - default: - logger.warn("No binning strategy has been set. Selecting BinningStrategyBinCount."); - this.binningStrategy = BinningStrategyFactory.getBinningStrategy(getQueryRange(), (int) binLengthOrCount); - } - } else { - this.binningStrategy = BinningStrategyFactory.getBinningStrategy(getQueryRange(), (int) binLengthOrCount); - } - } - - /** - * {@inheritDoc} - */ - @Override - public List getChannels() { - return Collections.unmodifiableList(channels); - } - - /** - * {@inheritDoc} - */ - @Override - public Ordering getOrdering() { - return ordering; - } - - /** - * {@inheritDoc} - */ - @Override - public AggregationType getAggregationType() { - return aggregationType; - } - - - @Override - public boolean isAggregateChannels() { - return aggregateChannels; - } - - public BinningStrategyEnum getBinningStrategyEnum() { - return binningStrategyEnum; - } - - @Override - public BinningStrategy getBinningStrategy() { - return binningStrategy; - } - - public LinkedHashSet getFields() { - return fields; - } - - public List getAggregations() { - return aggregations; - } - - /** - * {@inheritDoc} - */ - @Override - public QueryRange getQueryRange() { - return queryRange; - } - - public void setQueryRange(QueryRange queryRange) { - this.queryRange = queryRange; - } - - @Override - public String toString() { - return ReflectionToStringBuilder.reflectionToString(this); - } - -} diff --git a/src/main/java/ch/psi/daq/rest/queries/BinningStrategyEnum.java b/src/main/java/ch/psi/daq/rest/queries/BinningStrategyEnum.java deleted file mode 100644 index 08fcd45..0000000 --- a/src/main/java/ch/psi/daq/rest/queries/BinningStrategyEnum.java +++ /dev/null @@ -1,16 +0,0 @@ -package ch.psi.daq.rest.queries; - -import ch.psi.daq.hazelcast.query.bin.BinningStrategy; - -/** - * Defines the strategies that map to the similarly named {@link BinningStrategy} implementations. - *

- * This enum is used for type-safety and simplicity for the rest interface. - * - */ -public enum BinningStrategyEnum { - - length, - - count -} diff --git a/src/main/java/ch/psi/daq/rest/queries/PulseRangeQuery.java b/src/main/java/ch/psi/daq/rest/queries/PulseRangeQuery.java deleted file mode 100644 index 0237c95..0000000 --- a/src/main/java/ch/psi/daq/rest/queries/PulseRangeQuery.java +++ /dev/null @@ -1,69 +0,0 @@ -package ch.psi.daq.rest.queries; - -import java.util.LinkedHashSet; -import java.util.List; - -import org.apache.commons.lang3.builder.ToStringBuilder; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import ch.psi.daq.cassandra.reader.Ordering; -import ch.psi.daq.hazelcast.query.Aggregation; -import ch.psi.daq.hazelcast.query.AggregationType; -import ch.psi.daq.hazelcast.query.bin.BinningStrategy; -import ch.psi.daq.hazelcast.query.range.QueryRange; - -/** - * - */ -public class PulseRangeQuery extends AbstractQuery { - - - /** - * Constructor. - * - * @param ordering whether to add a 'orderBy' clause into the database query - * @param channels all the channelIds (channel names) we want to query - * @param fields the fields (who map to fields in the DB) we are interested in returning to the - * client, needs to be in insertion order (hence the {@link LinkedHashSet} type) - * @param binningStrategyEnum enum that maps the user's String to a concrete - * {@link BinningStrategy} implementation - * @param binLengthOrCount depending on the chosen binning strategy, this field defines either the - * count (how many pulse ids are to be put inside 1 bin) or the time frame for 1 bin - * @param aggregateChannels whether aggregation will include all channels, default is on a - * per-channel basis - * @param aggregationType defines whether aggregation takes place in an index- or value-based - * manner - * @param aggregations list of aggregations / statistics to calculate, e.g. min, max and average - * @param queryRange object containing the ranges for either pulse-based queries or time-based - * queries - */ - @JsonCreator - public PulseRangeQuery( - // note that those annotations are needed for the polymorphic - // mapping to work correctly - @JsonProperty(value = "ordering") Ordering ordering, - @JsonProperty(value = "channels") List channels, - @JsonProperty(value = "fields") LinkedHashSet fields, - @JsonProperty(value = "binningStrategy") BinningStrategyEnum binningStrategyEnum, - @JsonProperty(value = "binDuration") long binLengthOrCount, - @JsonProperty(value = "aggregateChannels") boolean aggregateChannels, - @JsonProperty(value = "aggregationType") AggregationType aggregationType, - @JsonProperty(value = "aggregations") List aggregations, - @JsonProperty(value = "queryRange") QueryRange queryRange) { - - super(ordering, channels, fields, binningStrategyEnum, binLengthOrCount, aggregateChannels, aggregationType, - aggregations, queryRange); - - } - - /** - * {@inheritDoc} - */ - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - -} diff --git a/src/main/java/ch/psi/daq/rest/queries/TimeRangeQuery.java b/src/main/java/ch/psi/daq/rest/queries/TimeRangeQuery.java deleted file mode 100644 index af4c1cc..0000000 --- a/src/main/java/ch/psi/daq/rest/queries/TimeRangeQuery.java +++ /dev/null @@ -1,96 +0,0 @@ -package ch.psi.daq.rest.queries; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.LinkedHashSet; -import java.util.List; - -import org.apache.commons.lang3.builder.ToStringBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import ch.psi.daq.cassandra.reader.Ordering; -import ch.psi.daq.hazelcast.query.Aggregation; -import ch.psi.daq.hazelcast.query.AggregationType; -import ch.psi.daq.hazelcast.query.bin.BinningStrategy; -import ch.psi.daq.hazelcast.query.range.QueryRange; - -public class TimeRangeQuery extends AbstractQuery { - - - public static final String DATE_FORMAT_STRING = "yyyy/MM/dd hh:mm:ss.SSS"; - private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat(DATE_FORMAT_STRING); - private static Logger logger = LoggerFactory.getLogger(TimeRangeQuery.class); - - /** - * Constructor - * - * @param ordering whether to add a 'orderBy' clause into the database query - * @param channels all the channelIds (channel names) we want to query - * @param fields the fields (who map to fields in the DB) we are interested in returning to the - * client, needs to be in insertion order (hence the {@link LinkedHashSet} type) - * @param binningStrategyEnum enum that maps the user's String to a concrete - * {@link BinningStrategy} implementation - * @param binLengthOrCount depending on the chosen binning strategy, this field defines either the - * count (how many pulse ids are to be put inside 1 bin) or the time frame for 1 bin - * @param aggregateChannels whether aggregation will include all channels, default is on a - * per-channel basis - * @param aggregationType defines whether aggregation takes place in an index- or value-based - * manner - * @param aggregations list of aggregations / statistics to calculate, e.g. min, max and average - * @param queryRange object containing the ranges for either pulse-based queries or time-based - * queries - * @param startDateTime if set, the date string (format is: - * {@link TimeRangeQuery#DATE_FORMAT_STRING} will be parsed and converted into - * milliseconds - * @param endDateTime set, the date string (format is: {@link TimeRangeQuery#DATE_FORMAT_STRING} - * will be parsed and converted into milliseconds - */ - @JsonCreator - public TimeRangeQuery( - // note that those annotations are needed for the polymorphic - // mapping to work correctly - @JsonProperty(value = "ordering") Ordering ordering, - @JsonProperty(value = "channels") List channels, - @JsonProperty(value = "fields") LinkedHashSet fields, - @JsonProperty(value = "binningStrategy") BinningStrategyEnum binningStrategyEnum, - @JsonProperty(value = "binDuration") long binLengthOrCount, - @JsonProperty(value = "aggregateChannels") boolean aggregateChannels, - @JsonProperty(value = "aggregationType") AggregationType aggregationType, - @JsonProperty(value = "aggregations") List aggregations, - @JsonProperty(value = "queryRange") QueryRange queryRange, - @JsonProperty(value = "startDateTime") String startDateTime, - @JsonProperty(value = "endDateTime") String endDateTime) { - - super(ordering, channels, fields, binningStrategyEnum, binLengthOrCount, aggregateChannels, aggregationType, - aggregations, queryRange); - - if (startDateTime != null && endDateTime != null) { - logger.info("startDateTime and endDateTime specified. This takes precedence over the start / end fields."); - try { - Date startDate = DATE_FORMAT.parse(startDateTime); - Date endDate = DATE_FORMAT.parse(endDateTime); - - getQueryRange().setTimeRange(startDate.getTime(), queryRange.getStartNanos(), endDate.getTime(), - queryRange.getEndNanos()); - } catch (ParseException e) { - logger.error("Parsing the start- and/or endDate was unsuccessful. " - + "The format must be '" + DATE_FORMAT_STRING + "'", e); - } - } - - } - - /** - * {@inheritDoc} - */ - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - -} diff --git a/src/main/java/ch/psi/daq/rest/ResponseStreamWriter.java b/src/main/java/ch/psi/daq/rest/response/ResponseStreamWriter.java similarity index 94% rename from src/main/java/ch/psi/daq/rest/ResponseStreamWriter.java rename to src/main/java/ch/psi/daq/rest/response/ResponseStreamWriter.java index 2d0efd5..11a2893 100644 --- a/src/main/java/ch/psi/daq/rest/ResponseStreamWriter.java +++ b/src/main/java/ch/psi/daq/rest/response/ResponseStreamWriter.java @@ -1,7 +1,7 @@ /** * */ -package ch.psi.daq.rest; +package ch.psi.daq.rest.response; import java.io.IOException; import java.util.Set; @@ -14,9 +14,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import ch.psi.daq.domain.cassandra.DataEvent; -import ch.psi.daq.rest.queries.AbstractQuery; - import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; @@ -25,6 +22,9 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter; import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider; +import ch.psi.daq.domain.cassandra.DataEvent; +import ch.psi.daq.hazelcast.query.AbstractQuery; + /** * Takes a Java 8 stream and writes it to the output stream provided by the {@link ServletResponse} * of the current request. @@ -39,6 +39,8 @@ public class ResponseStreamWriter { @Autowired private ObjectMapper mapper; + @Autowired + private Set defaultResponseFields; /** * Responding with the the contents of the stream by writing into the output stream of the * {@link ServletResponse}. @@ -51,6 +53,9 @@ public class ResponseStreamWriter { public void respond(Stream stream, AbstractQuery query, ServletResponse response) throws IOException { Set includedFields = query.getFields(); + if (includedFields == null) { + includedFields = defaultResponseFields; + } if (query.getAggregations() != null) { includedFields.addAll(query.getAggregations() diff --git a/src/main/resources/rest.properties b/src/main/resources/rest.properties index 9873024..2b5f02a 100644 --- a/src/main/resources/rest.properties +++ b/src/main/resources/rest.properties @@ -1,6 +1,6 @@ # port for the Spring boot application's embedded Tomcat server server.port=8080 -# defines the list of hosts who are tried for an initial connection to the cluster -#hazelcast.members=sf-nube-11.psi.ch,sf-nube-12.psi.ch,sf-nube-13.psi.ch,sf-nube-14.psi.ch -hazelcast.initialcandidates=localhost \ No newline at end of file +# defines the fields that are included in the response +# if no fields have been specified by the user +rest.default.response.fields=channel,pulseId,globalMillis,globalNanos,value \ No newline at end of file diff --git a/src/test/java/ch/psi/daq/test/rest/AbstractDaqRestTest.java b/src/test/java/ch/psi/daq/test/rest/AbstractDaqRestTest.java index 0ed239a..82453ca 100644 --- a/src/test/java/ch/psi/daq/test/rest/AbstractDaqRestTest.java +++ b/src/test/java/ch/psi/daq/test/rest/AbstractDaqRestTest.java @@ -24,7 +24,7 @@ import ch.psi.daq.test.cassandra.CassandraDaqUnitDependencyInjectionTestExecutio @TestExecutionListeners({ CassandraDaqUnitDependencyInjectionTestExecutionListener.class, DependencyInjectionTestExecutionListener.class}) -@SpringApplicationConfiguration(classes = {RestApplication.class, DaqWebMvcConfiguration.class}) +@SpringApplicationConfiguration(classes = {RestApplication.class, DaqWebMvcConfig.class}) //@EmbeddedCassandra @DirtiesContext(classMode = ClassMode.AFTER_EACH_TEST_METHOD) @WebAppConfiguration diff --git a/src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfig.java b/src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfig.java new file mode 100644 index 0000000..e799de1 --- /dev/null +++ b/src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfig.java @@ -0,0 +1,17 @@ +package ch.psi.daq.test.rest; + +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurationSupport; + +import ch.psi.daq.rest.config.RestConfig; +import ch.psi.daq.test.cassandra.LocalCassandraTestConfig; + +@Configuration +@Import(value = {LocalCassandraTestConfig.class, RestConfig.class}) +@EnableWebMvc +public class DaqWebMvcConfig extends WebMvcConfigurationSupport { + + // add test-specific beans and configurations here +} \ No newline at end of file diff --git a/src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfiguration.java b/src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfiguration.java deleted file mode 100644 index 0ab6185..0000000 --- a/src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfiguration.java +++ /dev/null @@ -1,24 +0,0 @@ -package ch.psi.daq.test.rest; - -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.web.servlet.config.annotation.EnableWebMvc; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurationSupport; - -import ch.psi.daq.rest.config.RestConfiguration; -import ch.psi.daq.test.cassandra.LocalCassandraTestConfig; - -@Configuration -//@ComponentScan(basePackages = { -// "ch.psi.daq.rest", -// "ch.psi.daq.cassandra.config", // define the package name with the CassandraConfig -// // configuration, or @Import it (see above) -// "ch.psi.daq.cassandra.reader", -// "ch.psi.daq.cassandra.writer" -//}) -@Import(value = {LocalCassandraTestConfig.class, RestConfiguration.class}) -@EnableWebMvc -public class DaqWebMvcConfiguration extends WebMvcConfigurationSupport { - - // add test-specific beans and configurations here -} \ No newline at end of file diff --git a/src/test/java/ch/psi/daq/test/rest/controller/DaqRestControllerTest.java b/src/test/java/ch/psi/daq/test/rest/controller/DaqRestControllerTest.java index d5960e3..8d9f47a 100644 --- a/src/test/java/ch/psi/daq/test/rest/controller/DaqRestControllerTest.java +++ b/src/test/java/ch/psi/daq/test/rest/controller/DaqRestControllerTest.java @@ -10,19 +10,19 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.test.web.servlet.result.MockMvcResultHandlers; import org.springframework.test.web.servlet.result.MockMvcResultMatchers; -import ch.psi.daq.cassandra.reader.Ordering; -import ch.psi.daq.hazelcast.query.AggregationType; -import ch.psi.daq.hazelcast.query.range.QueryRange; -import ch.psi.daq.hazelcast.query.range.QueryRangeImpl; -import ch.psi.daq.rest.queries.BinningStrategyEnum; -import ch.psi.daq.rest.queries.PulseRangeQuery; -import ch.psi.daq.rest.queries.TimeRangeQuery; -import ch.psi.daq.test.rest.AbstractDaqRestTest; - import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import ch.psi.daq.cassandra.reader.Ordering; +import ch.psi.daq.hazelcast.query.AggregationType; +import ch.psi.daq.hazelcast.query.BinningStrategyEnum; +import ch.psi.daq.hazelcast.query.PulseRangeQuery; +import ch.psi.daq.hazelcast.query.TimeRangeQuery; +import ch.psi.daq.hazelcast.query.range.QueryRange; +import ch.psi.daq.hazelcast.query.range.QueryRangeImpl; +import ch.psi.daq.test.rest.AbstractDaqRestTest; + /** * Tests the {@link DaqController} implementation. */ diff --git a/src/test/java/ch/psi/daq/test/rest/queries/AbstractQueryTestTest.java b/src/test/java/ch/psi/daq/test/rest/queries/AbstractQueryTestTest.java index 7475057..93b5e0a 100644 --- a/src/test/java/ch/psi/daq/test/rest/queries/AbstractQueryTestTest.java +++ b/src/test/java/ch/psi/daq/test/rest/queries/AbstractQueryTestTest.java @@ -8,18 +8,18 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.BlockJUnit4ClassRunner; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + import ch.psi.daq.cassandra.reader.Ordering; import ch.psi.daq.hazelcast.query.AggregationType; +import ch.psi.daq.hazelcast.query.BinningStrategyEnum; +import ch.psi.daq.hazelcast.query.TimeRangeQuery; import ch.psi.daq.hazelcast.query.bin.BinningStrategyBinCount; import ch.psi.daq.hazelcast.query.bin.BinningStrategyLengthPulse; import ch.psi.daq.hazelcast.query.bin.BinningStrategyLengthTime; import ch.psi.daq.hazelcast.query.range.QueryRange; import ch.psi.daq.hazelcast.query.range.QueryRangeImpl; -import ch.psi.daq.rest.queries.BinningStrategyEnum; -import ch.psi.daq.rest.queries.TimeRangeQuery; - -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; @RunWith(BlockJUnit4ClassRunner.class) public class AbstractQueryTestTest extends AbstractQueryTest { diff --git a/src/test/java/ch/psi/daq/test/rest/queries/TimeRangeQueryTest.java b/src/test/java/ch/psi/daq/test/rest/queries/TimeRangeQueryTest.java index d847dbd..931d90e 100644 --- a/src/test/java/ch/psi/daq/test/rest/queries/TimeRangeQueryTest.java +++ b/src/test/java/ch/psi/daq/test/rest/queries/TimeRangeQueryTest.java @@ -8,16 +8,16 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.BlockJUnit4ClassRunner; -import ch.psi.daq.cassandra.reader.Ordering; -import ch.psi.daq.hazelcast.query.AggregationType; -import ch.psi.daq.hazelcast.query.range.QueryRange; -import ch.psi.daq.hazelcast.query.range.QueryRangeImpl; -import ch.psi.daq.rest.queries.BinningStrategyEnum; -import ch.psi.daq.rest.queries.TimeRangeQuery; - import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import ch.psi.daq.cassandra.reader.Ordering; +import ch.psi.daq.hazelcast.query.AggregationType; +import ch.psi.daq.hazelcast.query.BinningStrategyEnum; +import ch.psi.daq.hazelcast.query.TimeRangeQuery; +import ch.psi.daq.hazelcast.query.range.QueryRange; +import ch.psi.daq.hazelcast.query.range.QueryRangeImpl; + @RunWith(BlockJUnit4ClassRunner.class) public class TimeRangeQueryTest extends AbstractQueryTest { diff --git a/src/test/java/ch/psi/daq/test/rest/query/DummyQueryProcessor.java b/src/test/java/ch/psi/daq/test/rest/query/DummyQueryProcessor.java index fd96466..0ffa7be 100644 --- a/src/test/java/ch/psi/daq/test/rest/query/DummyQueryProcessor.java +++ b/src/test/java/ch/psi/daq/test/rest/query/DummyQueryProcessor.java @@ -10,14 +10,14 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + import ch.psi.daq.domain.cassandra.ChannelEvent; import ch.psi.daq.domain.cassandra.DataEvent; import ch.psi.daq.hazelcast.query.Query; import ch.psi.daq.hazelcast.query.processor.QueryProcessor; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - /** * @author zellweger_c * diff --git a/src/test/resources/test-requests.txt b/src/test/resources/test-requests.txt index aca65fb..baafafe 100644 --- a/src/test/resources/test-requests.txt +++ b/src/test/resources/test-requests.txt @@ -73,4 +73,4 @@ curl -v -X POST -H 'Content-Type: application/json' -d '{"queryType":"timerange" -curl -v -X POST -H 'Content-Type: application/json' -d '{"queryType":"pulserange","ordering":"ASC","channels":["dummy-test"],"fields":["channel","pulseId","globalMillis","globalNanos","dbValueBytes"],"binningStrategy":"bincount","binDuration":100,"aggregateChannels":"false","aggregationType":"index","aggregations":[{"fieldRef":"e_val","type":"max","resultFieldName":"maximum"},{"fieldRef":"e_val","type":"min","resultFieldName":"minimum"}], "queryRange":{"startPulseId":100,"endPulseId":100}}' http://localhost:8080/pulserange \ No newline at end of file +curl -v -X POST -H 'Content-Type: application/json' -d '{"queryType":"pulserange","ordering":"ASC","channels":["dummy-test"],"fields":["channel","pulseId","globalMillis","globalNanos","dbValueBytes"],"binningStrategy":"bincount","binDuration":100,"aggregateChannels":"false","aggregationType":"index","aggregations":[{"fieldRef":"e_val","type":"max","resultFieldName":"maximum"},{"fieldRef":"e_val","type":"min","resultFieldName":"minimum"}], "queryRange":{"startPulseId":100,"endPulseId":100}}' http://localhost:8080/pulserange