From 8f2bbd47bd11cc966b929cb85615ca33ce962a75 Mon Sep 17 00:00:00 2001 From: Zellweger Christof Ralf Date: Fri, 26 Jun 2015 15:19:43 +0200 Subject: [PATCH 1/7] ATEST-81: - renaming config so that all the configs follow the same naming patterns --- .../rest/config/{RestConfiguration.java => RestConfig.java} | 4 ++-- .../java/ch/psi/daq/test/rest/DaqWebMvcConfiguration.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename src/main/java/ch/psi/daq/rest/config/{RestConfiguration.java => RestConfig.java} (98%) diff --git a/src/main/java/ch/psi/daq/rest/config/RestConfiguration.java b/src/main/java/ch/psi/daq/rest/config/RestConfig.java similarity index 98% rename from src/main/java/ch/psi/daq/rest/config/RestConfiguration.java rename to src/main/java/ch/psi/daq/rest/config/RestConfig.java index d904006..724ae67 100644 --- a/src/main/java/ch/psi/daq/rest/config/RestConfiguration.java +++ b/src/main/java/ch/psi/daq/rest/config/RestConfig.java @@ -32,9 +32,9 @@ import ch.psi.daq.rest.model.PropertyFilterMixin; @Configuration @PropertySource(value = { "classpath:rest.properties" }) @PropertySource(value = { "file:${user.home}/.config/daq/rest.properties" }, ignoreResourceNotFound = true) -public class RestConfiguration { +public class RestConfig { - private static final Logger logger = LoggerFactory.getLogger(RestConfiguration.class); + private static final Logger logger = LoggerFactory.getLogger(RestConfig.class); @Autowired private Environment env; diff --git a/src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfiguration.java b/src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfiguration.java index 0ab6185..6f82463 100644 --- a/src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfiguration.java +++ b/src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfiguration.java @@ -5,7 +5,7 @@ import org.springframework.context.annotation.Import; import org.springframework.web.servlet.config.annotation.EnableWebMvc; import org.springframework.web.servlet.config.annotation.WebMvcConfigurationSupport; -import ch.psi.daq.rest.config.RestConfiguration; +import ch.psi.daq.rest.config.RestConfig; import ch.psi.daq.test.cassandra.LocalCassandraTestConfig; @Configuration @@ -16,7 +16,7 @@ import ch.psi.daq.test.cassandra.LocalCassandraTestConfig; // "ch.psi.daq.cassandra.reader", // "ch.psi.daq.cassandra.writer" //}) -@Import(value = {LocalCassandraTestConfig.class, RestConfiguration.class}) +@Import(value = {LocalCassandraTestConfig.class, RestConfig.class}) @EnableWebMvc public class DaqWebMvcConfiguration extends WebMvcConfigurationSupport { From 1394a8b04cb0d243d2321849946aee2c05e74dfb Mon Sep 17 00:00:00 2001 From: Zellweger Christof Ralf Date: Tue, 30 Jun 2015 09:44:34 +0200 Subject: [PATCH 2/7] ATEST-81: - implementing dummy write method for manual testing --- .../java/ch/psi/daq/rest/RestApplication.java | 4 +- .../ch/psi/daq/rest/config/RestConfig.java | 40 +++++---- .../rest/controller/DaqRestController.java | 83 +++++++++++++++---- src/main/resources/rest.properties | 6 +- 4 files changed, 99 insertions(+), 34 deletions(-) diff --git a/src/main/java/ch/psi/daq/rest/RestApplication.java b/src/main/java/ch/psi/daq/rest/RestApplication.java index 86a2c04..10d391f 100644 --- a/src/main/java/ch/psi/daq/rest/RestApplication.java +++ b/src/main/java/ch/psi/daq/rest/RestApplication.java @@ -7,8 +7,8 @@ import org.springframework.boot.context.web.SpringBootServletInitializer; import org.springframework.context.annotation.ComponentScan; /** - * Entry point to our rest-frontend of the Swissfel application which most importantly wires all the @RestController - * annotated classes. + * Entry point to our rest-frontend of the data acquisition (DAQ) application which most importantly + * wires all the @RestController annotated classes. *

* * This acts as a @Configuration class for Spring. As such it has @ComponentScan annotation that diff --git a/src/main/java/ch/psi/daq/rest/config/RestConfig.java b/src/main/java/ch/psi/daq/rest/config/RestConfig.java index 724ae67..a9e73a5 100644 --- a/src/main/java/ch/psi/daq/rest/config/RestConfig.java +++ b/src/main/java/ch/psi/daq/rest/config/RestConfig.java @@ -24,8 +24,6 @@ import com.hazelcast.spring.context.SpringManagedContext; import ch.psi.daq.common.statistic.StorelessStatistics; import ch.psi.daq.domain.cassandra.ChannelEvent; import ch.psi.daq.hazelcast.config.HazelcastConfig; -import ch.psi.daq.hazelcast.query.processor.QueryProcessor; -import ch.psi.daq.hazelcast.query.processor.cassandra.CassandraQueryProcessorDistributed; import ch.psi.daq.rest.ResponseStreamWriter; import ch.psi.daq.rest.model.PropertyFilterMixin; @@ -36,14 +34,21 @@ public class RestConfig { private static final Logger logger = LoggerFactory.getLogger(RestConfig.class); + // TODO refactor - also defined in HazelcastNodeConfig + private static final String HAZELCAST_GROUP_PASSWORD = "hazelcast.group.password"; + private static final String HAZELCAST_GROUP_NAME = "hazelcast.group.name"; + @Autowired private Environment env; - @Bean - public QueryProcessor queryProcessor() { - return new CassandraQueryProcessorDistributed(); - } - + // a nested configuration + // this guarantees that the ordering of the properties file is as expected + // see: + // https://jira.spring.io/browse/SPR-10409?focusedCommentId=101393&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-101393 + // @Configuration + // @Import(CassandraConfig.class) + // static class InnerConfiguration { } + @Bean public JsonFactory jsonFactory() { return new JsonFactory(); @@ -63,7 +68,16 @@ public class RestConfig { ClientConfig config = new ClientConfig(); config.setManagedContext(managedContext()); - config.getNetworkConfig().setAddresses(hazelcastMembers()); + config.getNetworkConfig().setAddresses(hazelcastInitialCandidates()); + + String groupName = env.getProperty(HAZELCAST_GROUP_NAME); + if (groupName != null) { + config.getGroupConfig().setName(groupName); + } + String groupPassword = env.getProperty(HAZELCAST_GROUP_PASSWORD); + if (groupPassword != null) { + config.getGroupConfig().setPassword(groupPassword); + } return config; } @@ -73,8 +87,7 @@ public class RestConfig { return new SpringManagedContext(); } - @Bean - public List hazelcastMembers() { + private List hazelcastInitialCandidates() { List clusterMembers = Arrays.asList(StringUtils.commaDelimitedListToStringArray(env.getProperty("hazelcast.initialcandidates"))); logger.info("The following hosts have been defined to form a Hazelcast cluster: {}", clusterMembers); return clusterMembers; @@ -92,11 +105,4 @@ public class RestConfig { return mapper; } - // a nested configuration - // this guarantees that the ordering of the properties file is as expected - // see: - // https://jira.spring.io/browse/SPR-10409?focusedCommentId=101393&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-101393 - // @Configuration - // @Import(CassandraConfig.class) - // static class InnerConfiguration { } } diff --git a/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java b/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java index 99d34c6..2b2109c 100644 --- a/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java +++ b/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java @@ -1,9 +1,15 @@ package ch.psi.daq.rest.controller; import java.io.IOException; +import java.util.Arrays; +import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.LongFunction; +import java.util.function.LongUnaryOperator; +import java.util.stream.Collectors; +import java.util.stream.LongStream; import java.util.stream.Stream; import javax.servlet.http.HttpServletResponse; @@ -11,11 +17,13 @@ import javax.servlet.http.HttpServletResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.Assert; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import ch.psi.daq.cassandra.writer.CassandraWriter; +import ch.psi.daq.domain.DataType; import ch.psi.daq.domain.cassandra.ChannelEvent; import ch.psi.daq.domain.cassandra.DataEvent; import ch.psi.daq.hazelcast.query.processor.QueryProcessor; @@ -38,7 +46,6 @@ public class DaqRestController { @Autowired private QueryProcessor queryProcessor; - @RequestMapping(value = "/pulserange") public void pulseRange(@RequestBody PulseRangeQuery query, HttpServletResponse res) throws IOException { @@ -71,20 +78,68 @@ public class DaqRestController { responseStreamWriter.respond(flatStreams, query, res); } - + // ========================================================================================== + // TODO this is simply a method for initial / rudimentary testing - remove once further evolved @RequestMapping(value = "/write") - public void writeDummyEntry() { - int pulseId = 100; - ChannelEvent event = new ChannelEvent( - "dummy-test", - pulseId, - 0, - pulseId, - pulseId, - 0, - "data_" + UUID.randomUUID().toString()); + public long writeDummyEntry() { - CompletableFuture future = cassandraWriter.writeAsync(3, 0, event); + long startIndex = System.currentTimeMillis(); + + writeData(3,startIndex, 100, new String[]{"channel1", "channel2"}); + + return startIndex; + } + + private void writeData(int dataReplication, long startIndex, long nrOfElements, + String... channelNames) { + writeData(dataReplication, startIndex, nrOfElements, + i -> 2 * i, + i -> 2 * i, + i -> 2 * i, + i -> 2 * i, + i -> 2 * i, + i -> Long.valueOf(2 * i), + channelNames); + } + + private void writeData(int dataReplication, long startIndex, long nrOfElements, + LongUnaryOperator iocMillis, LongUnaryOperator iocNanos, LongUnaryOperator pulseIds, + LongUnaryOperator globalMillis, LongUnaryOperator globalNanos, LongFunction valueFunction, + String... channelNames) { + + Assert.notNull(channelNames); + + CompletableFuture future; + + List events = + Arrays.stream(channelNames) + .parallel() + .flatMap( + channelName -> { + Stream stream = + LongStream + .range(startIndex, startIndex + nrOfElements) + .parallel() + .mapToObj( + i -> { + Object value = valueFunction.apply(i); + return new ChannelEvent(channelName, + iocMillis.applyAsLong(i), + iocNanos.applyAsLong(i), + pulseIds.applyAsLong(i), + globalMillis.applyAsLong(i), + globalNanos.applyAsLong(i), + value, + DataType.getTypeName(value.getClass()) + ); + } + ); + + return stream; + }) + .collect(Collectors.toList()); + + future = cassandraWriter.writeAsync(dataReplication, (int) TimeUnit.HOURS.toSeconds(1), events); future.join(); } } diff --git a/src/main/resources/rest.properties b/src/main/resources/rest.properties index 9873024..7999931 100644 --- a/src/main/resources/rest.properties +++ b/src/main/resources/rest.properties @@ -3,4 +3,8 @@ server.port=8080 # defines the list of hosts who are tried for an initial connection to the cluster #hazelcast.members=sf-nube-11.psi.ch,sf-nube-12.psi.ch,sf-nube-13.psi.ch,sf-nube-14.psi.ch -hazelcast.initialcandidates=localhost \ No newline at end of file +hazelcast.initialcandidates=localhost + +# defines the cluster group and its password +hazelcast.group.name=QueryCluster +hazelcast.group.password=d9zT5h*4!KAHesxdnDm7 From df02fa4d4ca46673111174fe07336d7a3ecd086c Mon Sep 17 00:00:00 2001 From: Zellweger Christof Ralf Date: Tue, 30 Jun 2015 13:22:10 +0200 Subject: [PATCH 3/7] ATEST-81: - removing ComponentScan --- .../java/ch/psi/daq/rest/RestApplication.java | 32 ++-------- .../ch/psi/daq/rest/config/RestConfig.java | 60 ++----------------- .../rest/controller/DaqRestController.java | 2 +- .../{ => response}/ResponseStreamWriter.java | 2 +- src/main/resources/rest.properties | 8 --- .../daq/test/rest/AbstractDaqRestTest.java | 2 +- ...onfiguration.java => DaqWebMvcConfig.java} | 9 +-- 7 files changed, 14 insertions(+), 101 deletions(-) rename src/main/java/ch/psi/daq/rest/{ => response}/ResponseStreamWriter.java (99%) rename src/test/java/ch/psi/daq/test/rest/{DaqWebMvcConfiguration.java => DaqWebMvcConfig.java} (58%) diff --git a/src/main/java/ch/psi/daq/rest/RestApplication.java b/src/main/java/ch/psi/daq/rest/RestApplication.java index 10d391f..172290e 100644 --- a/src/main/java/ch/psi/daq/rest/RestApplication.java +++ b/src/main/java/ch/psi/daq/rest/RestApplication.java @@ -4,39 +4,17 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.boot.context.web.SpringBootServletInitializer; -import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Import; + +import ch.psi.daq.hazelcast.config.HazelcastClientConfig; +import ch.psi.daq.hazelcast.config.HazelcastConfig; /** * Entry point to our rest-frontend of the data acquisition (DAQ) application which most importantly * wires all the @RestController annotated classes. - *

- * - * This acts as a @Configuration class for Spring. As such it has @ComponentScan annotation that - * enables scanning for another Spring components in current package and its subpackages. - *

- * Another annotation is @EnableAutoConfiguration which tells Spring Boot to run autoconfiguration. - *

- * It also extends SpringBootServletInitializer which will configure Spring servlet for us, and - * overrides the configure() method to point to itself, so Spring can find the main configuration. - *

- * Finally, the main() method consists of single static call to SpringApplication.run(). - *

- * Methods annotated with @Bean are Java beans that are container-managed, i.e. managed by Spring. - * Whenever there are @Autowire, @Inject or similar annotations found in the code (which is being - * scanned through the @ComponentScan annotation), the container then knows how to create those - * beans and inject them accordingly. */ @SpringBootApplication -// @Import(CassandraConfig.class) // either define the context to be imported, or see ComponentScan -// comment below -@ComponentScan(basePackages = { - "ch.psi.daq.cassandra.config", // define the package name with the CassandraConfig - // configuration, or @Import it (see above) - "ch.psi.daq.cassandra.reader", - "ch.psi.daq.cassandra.writer", - "ch.psi.daq.hazelcast", - "ch.psi.daq.rest", -}) +@Import({HazelcastConfig.class, HazelcastClientConfig.class}) public class RestApplication extends SpringBootServletInitializer { diff --git a/src/main/java/ch/psi/daq/rest/config/RestConfig.java b/src/main/java/ch/psi/daq/rest/config/RestConfig.java index a9e73a5..caba027 100644 --- a/src/main/java/ch/psi/daq/rest/config/RestConfig.java +++ b/src/main/java/ch/psi/daq/rest/config/RestConfig.java @@ -1,43 +1,27 @@ package ch.psi.daq.rest.config; -import java.util.Arrays; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; import org.springframework.context.annotation.PropertySource; import org.springframework.core.env.Environment; -import org.springframework.util.StringUtils; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.ObjectMapper; -import com.hazelcast.client.HazelcastClient; -import com.hazelcast.client.config.ClientConfig; -import com.hazelcast.core.HazelcastInstance; -import com.hazelcast.core.ManagedContext; -import com.hazelcast.spring.context.SpringManagedContext; import ch.psi.daq.common.statistic.StorelessStatistics; import ch.psi.daq.domain.cassandra.ChannelEvent; import ch.psi.daq.hazelcast.config.HazelcastConfig; -import ch.psi.daq.rest.ResponseStreamWriter; import ch.psi.daq.rest.model.PropertyFilterMixin; +import ch.psi.daq.rest.response.ResponseStreamWriter; @Configuration @PropertySource(value = { "classpath:rest.properties" }) @PropertySource(value = { "file:${user.home}/.config/daq/rest.properties" }, ignoreResourceNotFound = true) public class RestConfig { - private static final Logger logger = LoggerFactory.getLogger(RestConfig.class); - - // TODO refactor - also defined in HazelcastNodeConfig - private static final String HAZELCAST_GROUP_PASSWORD = "hazelcast.group.password"; - private static final String HAZELCAST_GROUP_NAME = "hazelcast.group.name"; - @Autowired private Environment env; @@ -45,9 +29,9 @@ public class RestConfig { // this guarantees that the ordering of the properties file is as expected // see: // https://jira.spring.io/browse/SPR-10409?focusedCommentId=101393&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-101393 - // @Configuration - // @Import(CassandraConfig.class) - // static class InnerConfiguration { } + @Configuration + @Import({HazelcastConfig.class}) + static class InnerConfiguration { } @Bean public JsonFactory jsonFactory() { @@ -58,40 +42,6 @@ public class RestConfig { public ResponseStreamWriter responseStreamWriter() { return new ResponseStreamWriter(); } - - @Bean(name = HazelcastConfig.BEAN_NAME_HAZELCAST_CLIENT) - public HazelcastInstance hazelcastClientInstance() { - return HazelcastClient.newHazelcastClient(hazelcastClientConfig()); - } - - private ClientConfig hazelcastClientConfig() { - ClientConfig config = new ClientConfig(); - config.setManagedContext(managedContext()); - - config.getNetworkConfig().setAddresses(hazelcastInitialCandidates()); - - String groupName = env.getProperty(HAZELCAST_GROUP_NAME); - if (groupName != null) { - config.getGroupConfig().setName(groupName); - } - String groupPassword = env.getProperty(HAZELCAST_GROUP_PASSWORD); - if (groupPassword != null) { - config.getGroupConfig().setPassword(groupPassword); - } - - return config; - } - - - private ManagedContext managedContext() { - return new SpringManagedContext(); - } - - private List hazelcastInitialCandidates() { - List clusterMembers = Arrays.asList(StringUtils.commaDelimitedListToStringArray(env.getProperty("hazelcast.initialcandidates"))); - logger.info("The following hosts have been defined to form a Hazelcast cluster: {}", clusterMembers); - return clusterMembers; - } @Bean public ObjectMapper objectMapper() { diff --git a/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java b/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java index 2b2109c..22c6ee5 100644 --- a/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java +++ b/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java @@ -27,10 +27,10 @@ import ch.psi.daq.domain.DataType; import ch.psi.daq.domain.cassandra.ChannelEvent; import ch.psi.daq.domain.cassandra.DataEvent; import ch.psi.daq.hazelcast.query.processor.QueryProcessor; -import ch.psi.daq.rest.ResponseStreamWriter; import ch.psi.daq.rest.queries.AbstractQuery; import ch.psi.daq.rest.queries.PulseRangeQuery; import ch.psi.daq.rest.queries.TimeRangeQuery; +import ch.psi.daq.rest.response.ResponseStreamWriter; @RestController public class DaqRestController { diff --git a/src/main/java/ch/psi/daq/rest/ResponseStreamWriter.java b/src/main/java/ch/psi/daq/rest/response/ResponseStreamWriter.java similarity index 99% rename from src/main/java/ch/psi/daq/rest/ResponseStreamWriter.java rename to src/main/java/ch/psi/daq/rest/response/ResponseStreamWriter.java index 2d0efd5..bba5e4c 100644 --- a/src/main/java/ch/psi/daq/rest/ResponseStreamWriter.java +++ b/src/main/java/ch/psi/daq/rest/response/ResponseStreamWriter.java @@ -1,7 +1,7 @@ /** * */ -package ch.psi.daq.rest; +package ch.psi.daq.rest.response; import java.io.IOException; import java.util.Set; diff --git a/src/main/resources/rest.properties b/src/main/resources/rest.properties index 7999931..4e97ccf 100644 --- a/src/main/resources/rest.properties +++ b/src/main/resources/rest.properties @@ -1,10 +1,2 @@ # port for the Spring boot application's embedded Tomcat server server.port=8080 - -# defines the list of hosts who are tried for an initial connection to the cluster -#hazelcast.members=sf-nube-11.psi.ch,sf-nube-12.psi.ch,sf-nube-13.psi.ch,sf-nube-14.psi.ch -hazelcast.initialcandidates=localhost - -# defines the cluster group and its password -hazelcast.group.name=QueryCluster -hazelcast.group.password=d9zT5h*4!KAHesxdnDm7 diff --git a/src/test/java/ch/psi/daq/test/rest/AbstractDaqRestTest.java b/src/test/java/ch/psi/daq/test/rest/AbstractDaqRestTest.java index 0ed239a..82453ca 100644 --- a/src/test/java/ch/psi/daq/test/rest/AbstractDaqRestTest.java +++ b/src/test/java/ch/psi/daq/test/rest/AbstractDaqRestTest.java @@ -24,7 +24,7 @@ import ch.psi.daq.test.cassandra.CassandraDaqUnitDependencyInjectionTestExecutio @TestExecutionListeners({ CassandraDaqUnitDependencyInjectionTestExecutionListener.class, DependencyInjectionTestExecutionListener.class}) -@SpringApplicationConfiguration(classes = {RestApplication.class, DaqWebMvcConfiguration.class}) +@SpringApplicationConfiguration(classes = {RestApplication.class, DaqWebMvcConfig.class}) //@EmbeddedCassandra @DirtiesContext(classMode = ClassMode.AFTER_EACH_TEST_METHOD) @WebAppConfiguration diff --git a/src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfiguration.java b/src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfig.java similarity index 58% rename from src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfiguration.java rename to src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfig.java index 6f82463..e799de1 100644 --- a/src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfiguration.java +++ b/src/test/java/ch/psi/daq/test/rest/DaqWebMvcConfig.java @@ -9,16 +9,9 @@ import ch.psi.daq.rest.config.RestConfig; import ch.psi.daq.test.cassandra.LocalCassandraTestConfig; @Configuration -//@ComponentScan(basePackages = { -// "ch.psi.daq.rest", -// "ch.psi.daq.cassandra.config", // define the package name with the CassandraConfig -// // configuration, or @Import it (see above) -// "ch.psi.daq.cassandra.reader", -// "ch.psi.daq.cassandra.writer" -//}) @Import(value = {LocalCassandraTestConfig.class, RestConfig.class}) @EnableWebMvc -public class DaqWebMvcConfiguration extends WebMvcConfigurationSupport { +public class DaqWebMvcConfig extends WebMvcConfigurationSupport { // add test-specific beans and configurations here } \ No newline at end of file From 2d7c299ef700d12c9e2d7e2b8e6e27262df2d7cc Mon Sep 17 00:00:00 2001 From: Zellweger Christof Ralf Date: Tue, 30 Jun 2015 18:54:50 +0200 Subject: [PATCH 4/7] ATEST-81: - fixing / trying to fix various bugs --- src/main/java/ch/psi/daq/rest/queries/AbstractQuery.java | 2 ++ src/main/java/ch/psi/daq/rest/queries/PulseRangeQuery.java | 2 ++ src/main/java/ch/psi/daq/rest/queries/TimeRangeQuery.java | 3 ++- src/test/resources/test-requests.txt | 2 +- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/ch/psi/daq/rest/queries/AbstractQuery.java b/src/main/java/ch/psi/daq/rest/queries/AbstractQuery.java index bcf67c3..8f58c6d 100644 --- a/src/main/java/ch/psi/daq/rest/queries/AbstractQuery.java +++ b/src/main/java/ch/psi/daq/rest/queries/AbstractQuery.java @@ -39,6 +39,8 @@ import ch.psi.daq.hazelcast.query.range.QueryRange; }) public abstract class AbstractQuery implements Query { + private static final long serialVersionUID = 1L; + private static Logger logger = LoggerFactory.getLogger(AbstractQuery.class); private List channels; diff --git a/src/main/java/ch/psi/daq/rest/queries/PulseRangeQuery.java b/src/main/java/ch/psi/daq/rest/queries/PulseRangeQuery.java index 0237c95..87b5378 100644 --- a/src/main/java/ch/psi/daq/rest/queries/PulseRangeQuery.java +++ b/src/main/java/ch/psi/daq/rest/queries/PulseRangeQuery.java @@ -19,6 +19,8 @@ import ch.psi.daq.hazelcast.query.range.QueryRange; */ public class PulseRangeQuery extends AbstractQuery { + + private static final long serialVersionUID = 1L; /** * Constructor. diff --git a/src/main/java/ch/psi/daq/rest/queries/TimeRangeQuery.java b/src/main/java/ch/psi/daq/rest/queries/TimeRangeQuery.java index af4c1cc..53f42ae 100644 --- a/src/main/java/ch/psi/daq/rest/queries/TimeRangeQuery.java +++ b/src/main/java/ch/psi/daq/rest/queries/TimeRangeQuery.java @@ -20,7 +20,8 @@ import ch.psi.daq.hazelcast.query.bin.BinningStrategy; import ch.psi.daq.hazelcast.query.range.QueryRange; public class TimeRangeQuery extends AbstractQuery { - + + private static final long serialVersionUID = 1L; public static final String DATE_FORMAT_STRING = "yyyy/MM/dd hh:mm:ss.SSS"; private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat(DATE_FORMAT_STRING); diff --git a/src/test/resources/test-requests.txt b/src/test/resources/test-requests.txt index aca65fb..baafafe 100644 --- a/src/test/resources/test-requests.txt +++ b/src/test/resources/test-requests.txt @@ -73,4 +73,4 @@ curl -v -X POST -H 'Content-Type: application/json' -d '{"queryType":"timerange" -curl -v -X POST -H 'Content-Type: application/json' -d '{"queryType":"pulserange","ordering":"ASC","channels":["dummy-test"],"fields":["channel","pulseId","globalMillis","globalNanos","dbValueBytes"],"binningStrategy":"bincount","binDuration":100,"aggregateChannels":"false","aggregationType":"index","aggregations":[{"fieldRef":"e_val","type":"max","resultFieldName":"maximum"},{"fieldRef":"e_val","type":"min","resultFieldName":"minimum"}], "queryRange":{"startPulseId":100,"endPulseId":100}}' http://localhost:8080/pulserange \ No newline at end of file +curl -v -X POST -H 'Content-Type: application/json' -d '{"queryType":"pulserange","ordering":"ASC","channels":["dummy-test"],"fields":["channel","pulseId","globalMillis","globalNanos","dbValueBytes"],"binningStrategy":"bincount","binDuration":100,"aggregateChannels":"false","aggregationType":"index","aggregations":[{"fieldRef":"e_val","type":"max","resultFieldName":"maximum"},{"fieldRef":"e_val","type":"min","resultFieldName":"minimum"}], "queryRange":{"startPulseId":100,"endPulseId":100}}' http://localhost:8080/pulserange From 6dfa14082d68a154c7ce7753cd5d57924d4b382a Mon Sep 17 00:00:00 2001 From: Zellweger Christof Ralf Date: Wed, 1 Jul 2015 16:51:22 +0200 Subject: [PATCH 5/7] ATEST-81: - tracking down bugs and trying to fix --- .../java/ch/psi/daq/rest/RestApplication.java | 5 - .../ch/psi/daq/rest/config/RestConfig.java | 18 +- .../rest/controller/DaqRestController.java | 6 +- .../psi/daq/rest/queries/AbstractQuery.java | 194 ------------------ .../daq/rest/queries/BinningStrategyEnum.java | 16 -- .../psi/daq/rest/queries/PulseRangeQuery.java | 71 ------- .../psi/daq/rest/queries/TimeRangeQuery.java | 97 --------- .../rest/response/ResponseStreamWriter.java | 6 +- .../controller/DaqRestControllerTest.java | 18 +- .../rest/queries/AbstractQueryTestTest.java | 10 +- .../test/rest/queries/TimeRangeQueryTest.java | 14 +- .../test/rest/query/DummyQueryProcessor.java | 6 +- 12 files changed, 40 insertions(+), 421 deletions(-) delete mode 100644 src/main/java/ch/psi/daq/rest/queries/AbstractQuery.java delete mode 100644 src/main/java/ch/psi/daq/rest/queries/BinningStrategyEnum.java delete mode 100644 src/main/java/ch/psi/daq/rest/queries/PulseRangeQuery.java delete mode 100644 src/main/java/ch/psi/daq/rest/queries/TimeRangeQuery.java diff --git a/src/main/java/ch/psi/daq/rest/RestApplication.java b/src/main/java/ch/psi/daq/rest/RestApplication.java index 172290e..f61fce4 100644 --- a/src/main/java/ch/psi/daq/rest/RestApplication.java +++ b/src/main/java/ch/psi/daq/rest/RestApplication.java @@ -4,17 +4,12 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.boot.context.web.SpringBootServletInitializer; -import org.springframework.context.annotation.Import; - -import ch.psi.daq.hazelcast.config.HazelcastClientConfig; -import ch.psi.daq.hazelcast.config.HazelcastConfig; /** * Entry point to our rest-frontend of the data acquisition (DAQ) application which most importantly * wires all the @RestController annotated classes. */ @SpringBootApplication -@Import({HazelcastConfig.class, HazelcastClientConfig.class}) public class RestApplication extends SpringBootServletInitializer { diff --git a/src/main/java/ch/psi/daq/rest/config/RestConfig.java b/src/main/java/ch/psi/daq/rest/config/RestConfig.java index caba027..9aa21b3 100644 --- a/src/main/java/ch/psi/daq/rest/config/RestConfig.java +++ b/src/main/java/ch/psi/daq/rest/config/RestConfig.java @@ -13,31 +13,33 @@ import com.fasterxml.jackson.databind.ObjectMapper; import ch.psi.daq.common.statistic.StorelessStatistics; import ch.psi.daq.domain.cassandra.ChannelEvent; +import ch.psi.daq.hazelcast.config.HazelcastClientConfig; import ch.psi.daq.hazelcast.config.HazelcastConfig; import ch.psi.daq.rest.model.PropertyFilterMixin; import ch.psi.daq.rest.response.ResponseStreamWriter; @Configuration -@PropertySource(value = { "classpath:rest.properties" }) -@PropertySource(value = { "file:${user.home}/.config/daq/rest.properties" }, ignoreResourceNotFound = true) +@PropertySource(value = {"classpath:rest.properties"}) +@PropertySource(value = {"file:${user.home}/.config/daq/rest.properties"}, ignoreResourceNotFound = true) public class RestConfig { @Autowired private Environment env; - + // a nested configuration // this guarantees that the ordering of the properties file is as expected // see: // https://jira.spring.io/browse/SPR-10409?focusedCommentId=101393&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-101393 - @Configuration - @Import({HazelcastConfig.class}) - static class InnerConfiguration { } - + @Configuration + @Import({HazelcastConfig.class, HazelcastClientConfig.class}) + static class InnerConfiguration { + } + @Bean public JsonFactory jsonFactory() { return new JsonFactory(); } - + @Bean public ResponseStreamWriter responseStreamWriter() { return new ResponseStreamWriter(); diff --git a/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java b/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java index 22c6ee5..8b537e8 100644 --- a/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java +++ b/src/main/java/ch/psi/daq/rest/controller/DaqRestController.java @@ -26,10 +26,10 @@ import ch.psi.daq.cassandra.writer.CassandraWriter; import ch.psi.daq.domain.DataType; import ch.psi.daq.domain.cassandra.ChannelEvent; import ch.psi.daq.domain.cassandra.DataEvent; +import ch.psi.daq.hazelcast.query.AbstractQuery; +import ch.psi.daq.hazelcast.query.PulseRangeQuery; +import ch.psi.daq.hazelcast.query.TimeRangeQuery; import ch.psi.daq.hazelcast.query.processor.QueryProcessor; -import ch.psi.daq.rest.queries.AbstractQuery; -import ch.psi.daq.rest.queries.PulseRangeQuery; -import ch.psi.daq.rest.queries.TimeRangeQuery; import ch.psi.daq.rest.response.ResponseStreamWriter; @RestController diff --git a/src/main/java/ch/psi/daq/rest/queries/AbstractQuery.java b/src/main/java/ch/psi/daq/rest/queries/AbstractQuery.java deleted file mode 100644 index 8f58c6d..0000000 --- a/src/main/java/ch/psi/daq/rest/queries/AbstractQuery.java +++ /dev/null @@ -1,194 +0,0 @@ -package ch.psi.daq.rest.queries; - -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; - -import org.apache.commons.lang3.builder.ReflectionToStringBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.Assert; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonSubTypes.Type; -import com.fasterxml.jackson.annotation.JsonTypeInfo; - -import ch.psi.daq.cassandra.reader.Ordering; -import ch.psi.daq.hazelcast.query.Aggregation; -import ch.psi.daq.hazelcast.query.AggregationType; -import ch.psi.daq.hazelcast.query.Query; -import ch.psi.daq.hazelcast.query.bin.BinningStrategy; -import ch.psi.daq.hazelcast.query.bin.BinningStrategyFactory; -import ch.psi.daq.hazelcast.query.range.QueryRange; - -/** - * - * @author zellweger_c - * - */ -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "queryType") -@JsonSubTypes( -{ - @Type(value = PulseRangeQuery.class, name = "pulserange"), - @Type(value = TimeRangeQuery.class, name = "timerange"), -}) -public abstract class AbstractQuery implements Query { - - private static final long serialVersionUID = 1L; - - private static Logger logger = LoggerFactory.getLogger(AbstractQuery.class); - - private List channels; - - private LinkedHashSet fields; - - private List aggregations; - - private AggregationType aggregationType; - - private boolean aggregateChannels; - - private Ordering ordering; - - private BinningStrategy binningStrategy; - - private BinningStrategyEnum binningStrategyEnum; - - private QueryRange queryRange; - - /** - * Constructor. - * - * @param ordering whether to add a 'orderBy' clause into the database query - * @param channels all the channelIds (channel names) we want to query - * @param fields the fields (who map to fields in the DB) we are interested in returning to the - * client, needs to be in insertion order (hence the {@link LinkedHashSet} type) - * @param binningStrategyEnum enum that maps the user's String to a concrete - * {@link BinningStrategy} implementation - * @param binLengthOrCount depending on the chosen binning strategy, this field defines either the - * count (how many pulse ids are to be put inside 1 bin) or the time frame for 1 bin - * @param aggregateChannels whether aggregation will include all channels, default is on a - * per-channel basis - * @param aggregationType defines whether aggregation takes place in an index- or value-based - * manner - * @param aggregations list of aggregations / statistics to calculate, e.g. min, max and average - * @param queryRange object containing the ranges for either pulse-based queries or time-based - * queries - */ - @JsonCreator - public AbstractQuery( - // note that those annotations are needed for the polymorphic - // mapping to work correctly - @JsonProperty(value = "ordering") Ordering ordering, - @JsonProperty(value = "channels") List channels, - @JsonProperty(value = "fields") LinkedHashSet fields, - @JsonProperty(value = "binningStrategy") BinningStrategyEnum binningStrategyEnum, - @JsonProperty(value = "binDuration") long binLengthOrCount, - @JsonProperty(value = "aggregateChannels") boolean aggregateChannels, - @JsonProperty(value = "aggregationType") AggregationType aggregationType, - @JsonProperty(value = "aggregations") List aggregations, - @JsonProperty(value = "queryRange") QueryRange queryRange) { - - this.ordering = ordering; - this.aggregateChannels = aggregateChannels; - this.aggregationType = aggregationType; - this.aggregations = aggregations; - this.queryRange = queryRange; - - Assert.notNull(channels, "channel name must not be null."); - Assert.notNull(fields, "field, i.e. property, names must not be null."); - - this.channels = channels; - this.fields = fields; - - this.binningStrategyEnum = binningStrategyEnum; // can be null: default then will be - // BinCountBinningStrategy - - - if (binningStrategyEnum != null) { - switch (binningStrategyEnum) { - case count: - this.binningStrategy = BinningStrategyFactory.getBinningStrategy(getQueryRange(), (int) binLengthOrCount); - break; - case length: - this.binningStrategy = BinningStrategyFactory.getBinningStrategy(getQueryRange(), binLengthOrCount); - break; - default: - logger.warn("No binning strategy has been set. Selecting BinningStrategyBinCount."); - this.binningStrategy = BinningStrategyFactory.getBinningStrategy(getQueryRange(), (int) binLengthOrCount); - } - } else { - this.binningStrategy = BinningStrategyFactory.getBinningStrategy(getQueryRange(), (int) binLengthOrCount); - } - } - - /** - * {@inheritDoc} - */ - @Override - public List getChannels() { - return Collections.unmodifiableList(channels); - } - - /** - * {@inheritDoc} - */ - @Override - public Ordering getOrdering() { - return ordering; - } - - /** - * {@inheritDoc} - */ - @Override - public AggregationType getAggregationType() { - return aggregationType; - } - - - @Override - public boolean isAggregateChannels() { - return aggregateChannels; - } - - public BinningStrategyEnum getBinningStrategyEnum() { - return binningStrategyEnum; - } - - @Override - public BinningStrategy getBinningStrategy() { - return binningStrategy; - } - - public LinkedHashSet getFields() { - return fields; - } - - public List getAggregations() { - return aggregations; - } - - /** - * {@inheritDoc} - */ - @Override - public QueryRange getQueryRange() { - return queryRange; - } - - public void setQueryRange(QueryRange queryRange) { - this.queryRange = queryRange; - } - - @Override - public String toString() { - return ReflectionToStringBuilder.reflectionToString(this); - } - -} diff --git a/src/main/java/ch/psi/daq/rest/queries/BinningStrategyEnum.java b/src/main/java/ch/psi/daq/rest/queries/BinningStrategyEnum.java deleted file mode 100644 index 08fcd45..0000000 --- a/src/main/java/ch/psi/daq/rest/queries/BinningStrategyEnum.java +++ /dev/null @@ -1,16 +0,0 @@ -package ch.psi.daq.rest.queries; - -import ch.psi.daq.hazelcast.query.bin.BinningStrategy; - -/** - * Defines the strategies that map to the similarly named {@link BinningStrategy} implementations. - *

- * This enum is used for type-safety and simplicity for the rest interface. - * - */ -public enum BinningStrategyEnum { - - length, - - count -} diff --git a/src/main/java/ch/psi/daq/rest/queries/PulseRangeQuery.java b/src/main/java/ch/psi/daq/rest/queries/PulseRangeQuery.java deleted file mode 100644 index 87b5378..0000000 --- a/src/main/java/ch/psi/daq/rest/queries/PulseRangeQuery.java +++ /dev/null @@ -1,71 +0,0 @@ -package ch.psi.daq.rest.queries; - -import java.util.LinkedHashSet; -import java.util.List; - -import org.apache.commons.lang3.builder.ToStringBuilder; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import ch.psi.daq.cassandra.reader.Ordering; -import ch.psi.daq.hazelcast.query.Aggregation; -import ch.psi.daq.hazelcast.query.AggregationType; -import ch.psi.daq.hazelcast.query.bin.BinningStrategy; -import ch.psi.daq.hazelcast.query.range.QueryRange; - -/** - * - */ -public class PulseRangeQuery extends AbstractQuery { - - - private static final long serialVersionUID = 1L; - - /** - * Constructor. - * - * @param ordering whether to add a 'orderBy' clause into the database query - * @param channels all the channelIds (channel names) we want to query - * @param fields the fields (who map to fields in the DB) we are interested in returning to the - * client, needs to be in insertion order (hence the {@link LinkedHashSet} type) - * @param binningStrategyEnum enum that maps the user's String to a concrete - * {@link BinningStrategy} implementation - * @param binLengthOrCount depending on the chosen binning strategy, this field defines either the - * count (how many pulse ids are to be put inside 1 bin) or the time frame for 1 bin - * @param aggregateChannels whether aggregation will include all channels, default is on a - * per-channel basis - * @param aggregationType defines whether aggregation takes place in an index- or value-based - * manner - * @param aggregations list of aggregations / statistics to calculate, e.g. min, max and average - * @param queryRange object containing the ranges for either pulse-based queries or time-based - * queries - */ - @JsonCreator - public PulseRangeQuery( - // note that those annotations are needed for the polymorphic - // mapping to work correctly - @JsonProperty(value = "ordering") Ordering ordering, - @JsonProperty(value = "channels") List channels, - @JsonProperty(value = "fields") LinkedHashSet fields, - @JsonProperty(value = "binningStrategy") BinningStrategyEnum binningStrategyEnum, - @JsonProperty(value = "binDuration") long binLengthOrCount, - @JsonProperty(value = "aggregateChannels") boolean aggregateChannels, - @JsonProperty(value = "aggregationType") AggregationType aggregationType, - @JsonProperty(value = "aggregations") List aggregations, - @JsonProperty(value = "queryRange") QueryRange queryRange) { - - super(ordering, channels, fields, binningStrategyEnum, binLengthOrCount, aggregateChannels, aggregationType, - aggregations, queryRange); - - } - - /** - * {@inheritDoc} - */ - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - -} diff --git a/src/main/java/ch/psi/daq/rest/queries/TimeRangeQuery.java b/src/main/java/ch/psi/daq/rest/queries/TimeRangeQuery.java deleted file mode 100644 index 53f42ae..0000000 --- a/src/main/java/ch/psi/daq/rest/queries/TimeRangeQuery.java +++ /dev/null @@ -1,97 +0,0 @@ -package ch.psi.daq.rest.queries; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.LinkedHashSet; -import java.util.List; - -import org.apache.commons.lang3.builder.ToStringBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import ch.psi.daq.cassandra.reader.Ordering; -import ch.psi.daq.hazelcast.query.Aggregation; -import ch.psi.daq.hazelcast.query.AggregationType; -import ch.psi.daq.hazelcast.query.bin.BinningStrategy; -import ch.psi.daq.hazelcast.query.range.QueryRange; - -public class TimeRangeQuery extends AbstractQuery { - - private static final long serialVersionUID = 1L; - - public static final String DATE_FORMAT_STRING = "yyyy/MM/dd hh:mm:ss.SSS"; - private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat(DATE_FORMAT_STRING); - private static Logger logger = LoggerFactory.getLogger(TimeRangeQuery.class); - - /** - * Constructor - * - * @param ordering whether to add a 'orderBy' clause into the database query - * @param channels all the channelIds (channel names) we want to query - * @param fields the fields (who map to fields in the DB) we are interested in returning to the - * client, needs to be in insertion order (hence the {@link LinkedHashSet} type) - * @param binningStrategyEnum enum that maps the user's String to a concrete - * {@link BinningStrategy} implementation - * @param binLengthOrCount depending on the chosen binning strategy, this field defines either the - * count (how many pulse ids are to be put inside 1 bin) or the time frame for 1 bin - * @param aggregateChannels whether aggregation will include all channels, default is on a - * per-channel basis - * @param aggregationType defines whether aggregation takes place in an index- or value-based - * manner - * @param aggregations list of aggregations / statistics to calculate, e.g. min, max and average - * @param queryRange object containing the ranges for either pulse-based queries or time-based - * queries - * @param startDateTime if set, the date string (format is: - * {@link TimeRangeQuery#DATE_FORMAT_STRING} will be parsed and converted into - * milliseconds - * @param endDateTime set, the date string (format is: {@link TimeRangeQuery#DATE_FORMAT_STRING} - * will be parsed and converted into milliseconds - */ - @JsonCreator - public TimeRangeQuery( - // note that those annotations are needed for the polymorphic - // mapping to work correctly - @JsonProperty(value = "ordering") Ordering ordering, - @JsonProperty(value = "channels") List channels, - @JsonProperty(value = "fields") LinkedHashSet fields, - @JsonProperty(value = "binningStrategy") BinningStrategyEnum binningStrategyEnum, - @JsonProperty(value = "binDuration") long binLengthOrCount, - @JsonProperty(value = "aggregateChannels") boolean aggregateChannels, - @JsonProperty(value = "aggregationType") AggregationType aggregationType, - @JsonProperty(value = "aggregations") List aggregations, - @JsonProperty(value = "queryRange") QueryRange queryRange, - @JsonProperty(value = "startDateTime") String startDateTime, - @JsonProperty(value = "endDateTime") String endDateTime) { - - super(ordering, channels, fields, binningStrategyEnum, binLengthOrCount, aggregateChannels, aggregationType, - aggregations, queryRange); - - if (startDateTime != null && endDateTime != null) { - logger.info("startDateTime and endDateTime specified. This takes precedence over the start / end fields."); - try { - Date startDate = DATE_FORMAT.parse(startDateTime); - Date endDate = DATE_FORMAT.parse(endDateTime); - - getQueryRange().setTimeRange(startDate.getTime(), queryRange.getStartNanos(), endDate.getTime(), - queryRange.getEndNanos()); - } catch (ParseException e) { - logger.error("Parsing the start- and/or endDate was unsuccessful. " - + "The format must be '" + DATE_FORMAT_STRING + "'", e); - } - } - - } - - /** - * {@inheritDoc} - */ - @Override - public String toString() { - return ToStringBuilder.reflectionToString(this); - } - -} diff --git a/src/main/java/ch/psi/daq/rest/response/ResponseStreamWriter.java b/src/main/java/ch/psi/daq/rest/response/ResponseStreamWriter.java index bba5e4c..3a4963a 100644 --- a/src/main/java/ch/psi/daq/rest/response/ResponseStreamWriter.java +++ b/src/main/java/ch/psi/daq/rest/response/ResponseStreamWriter.java @@ -14,9 +14,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import ch.psi.daq.domain.cassandra.DataEvent; -import ch.psi.daq.rest.queries.AbstractQuery; - import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; @@ -25,6 +22,9 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter; import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider; +import ch.psi.daq.domain.cassandra.DataEvent; +import ch.psi.daq.hazelcast.query.AbstractQuery; + /** * Takes a Java 8 stream and writes it to the output stream provided by the {@link ServletResponse} * of the current request. diff --git a/src/test/java/ch/psi/daq/test/rest/controller/DaqRestControllerTest.java b/src/test/java/ch/psi/daq/test/rest/controller/DaqRestControllerTest.java index d5960e3..8d9f47a 100644 --- a/src/test/java/ch/psi/daq/test/rest/controller/DaqRestControllerTest.java +++ b/src/test/java/ch/psi/daq/test/rest/controller/DaqRestControllerTest.java @@ -10,19 +10,19 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.test.web.servlet.result.MockMvcResultHandlers; import org.springframework.test.web.servlet.result.MockMvcResultMatchers; -import ch.psi.daq.cassandra.reader.Ordering; -import ch.psi.daq.hazelcast.query.AggregationType; -import ch.psi.daq.hazelcast.query.range.QueryRange; -import ch.psi.daq.hazelcast.query.range.QueryRangeImpl; -import ch.psi.daq.rest.queries.BinningStrategyEnum; -import ch.psi.daq.rest.queries.PulseRangeQuery; -import ch.psi.daq.rest.queries.TimeRangeQuery; -import ch.psi.daq.test.rest.AbstractDaqRestTest; - import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import ch.psi.daq.cassandra.reader.Ordering; +import ch.psi.daq.hazelcast.query.AggregationType; +import ch.psi.daq.hazelcast.query.BinningStrategyEnum; +import ch.psi.daq.hazelcast.query.PulseRangeQuery; +import ch.psi.daq.hazelcast.query.TimeRangeQuery; +import ch.psi.daq.hazelcast.query.range.QueryRange; +import ch.psi.daq.hazelcast.query.range.QueryRangeImpl; +import ch.psi.daq.test.rest.AbstractDaqRestTest; + /** * Tests the {@link DaqController} implementation. */ diff --git a/src/test/java/ch/psi/daq/test/rest/queries/AbstractQueryTestTest.java b/src/test/java/ch/psi/daq/test/rest/queries/AbstractQueryTestTest.java index 7475057..93b5e0a 100644 --- a/src/test/java/ch/psi/daq/test/rest/queries/AbstractQueryTestTest.java +++ b/src/test/java/ch/psi/daq/test/rest/queries/AbstractQueryTestTest.java @@ -8,18 +8,18 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.BlockJUnit4ClassRunner; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + import ch.psi.daq.cassandra.reader.Ordering; import ch.psi.daq.hazelcast.query.AggregationType; +import ch.psi.daq.hazelcast.query.BinningStrategyEnum; +import ch.psi.daq.hazelcast.query.TimeRangeQuery; import ch.psi.daq.hazelcast.query.bin.BinningStrategyBinCount; import ch.psi.daq.hazelcast.query.bin.BinningStrategyLengthPulse; import ch.psi.daq.hazelcast.query.bin.BinningStrategyLengthTime; import ch.psi.daq.hazelcast.query.range.QueryRange; import ch.psi.daq.hazelcast.query.range.QueryRangeImpl; -import ch.psi.daq.rest.queries.BinningStrategyEnum; -import ch.psi.daq.rest.queries.TimeRangeQuery; - -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; @RunWith(BlockJUnit4ClassRunner.class) public class AbstractQueryTestTest extends AbstractQueryTest { diff --git a/src/test/java/ch/psi/daq/test/rest/queries/TimeRangeQueryTest.java b/src/test/java/ch/psi/daq/test/rest/queries/TimeRangeQueryTest.java index d847dbd..931d90e 100644 --- a/src/test/java/ch/psi/daq/test/rest/queries/TimeRangeQueryTest.java +++ b/src/test/java/ch/psi/daq/test/rest/queries/TimeRangeQueryTest.java @@ -8,16 +8,16 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.BlockJUnit4ClassRunner; -import ch.psi.daq.cassandra.reader.Ordering; -import ch.psi.daq.hazelcast.query.AggregationType; -import ch.psi.daq.hazelcast.query.range.QueryRange; -import ch.psi.daq.hazelcast.query.range.QueryRangeImpl; -import ch.psi.daq.rest.queries.BinningStrategyEnum; -import ch.psi.daq.rest.queries.TimeRangeQuery; - import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import ch.psi.daq.cassandra.reader.Ordering; +import ch.psi.daq.hazelcast.query.AggregationType; +import ch.psi.daq.hazelcast.query.BinningStrategyEnum; +import ch.psi.daq.hazelcast.query.TimeRangeQuery; +import ch.psi.daq.hazelcast.query.range.QueryRange; +import ch.psi.daq.hazelcast.query.range.QueryRangeImpl; + @RunWith(BlockJUnit4ClassRunner.class) public class TimeRangeQueryTest extends AbstractQueryTest { diff --git a/src/test/java/ch/psi/daq/test/rest/query/DummyQueryProcessor.java b/src/test/java/ch/psi/daq/test/rest/query/DummyQueryProcessor.java index fd96466..0ffa7be 100644 --- a/src/test/java/ch/psi/daq/test/rest/query/DummyQueryProcessor.java +++ b/src/test/java/ch/psi/daq/test/rest/query/DummyQueryProcessor.java @@ -10,14 +10,14 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + import ch.psi.daq.domain.cassandra.ChannelEvent; import ch.psi.daq.domain.cassandra.DataEvent; import ch.psi.daq.hazelcast.query.Query; import ch.psi.daq.hazelcast.query.processor.QueryProcessor; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - /** * @author zellweger_c * From 31570822628c2a081034f476bf697344822b80a7 Mon Sep 17 00:00:00 2001 From: Zellweger Christof Ralf Date: Fri, 3 Jul 2015 16:04:14 +0200 Subject: [PATCH 6/7] ATEST-81: - trying to fix bug re react stream and closed queue --- .../java/ch/psi/daq/rest/config/RestConfig.java | 17 ++++++++++++++++- .../daq/rest/response/ResponseStreamWriter.java | 5 +++++ src/main/resources/rest.properties | 4 ++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/main/java/ch/psi/daq/rest/config/RestConfig.java b/src/main/java/ch/psi/daq/rest/config/RestConfig.java index 9aa21b3..7b30cdf 100644 --- a/src/main/java/ch/psi/daq/rest/config/RestConfig.java +++ b/src/main/java/ch/psi/daq/rest/config/RestConfig.java @@ -1,15 +1,21 @@ package ch.psi.daq.rest.config; +import java.util.Arrays; +import java.util.List; +import java.util.Set; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.PropertySource; import org.springframework.core.env.Environment; +import org.springframework.util.StringUtils; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; import ch.psi.daq.common.statistic.StorelessStatistics; import ch.psi.daq.domain.cassandra.ChannelEvent; @@ -56,5 +62,14 @@ public class RestConfig { mapper.addMixIn(StorelessStatistics.class, PropertyFilterMixin.class); return mapper; } - + + @Bean + private Set defaultResponseFields() { + List defaultFields = Arrays.asList( + StringUtils.commaDelimitedListToStringArray( + env.getProperty("rest.default.response.fields") + )); + Set defaultResponseFields = Sets.newHashSet(defaultFields.iterator()); + return defaultResponseFields; + } } diff --git a/src/main/java/ch/psi/daq/rest/response/ResponseStreamWriter.java b/src/main/java/ch/psi/daq/rest/response/ResponseStreamWriter.java index 3a4963a..11a2893 100644 --- a/src/main/java/ch/psi/daq/rest/response/ResponseStreamWriter.java +++ b/src/main/java/ch/psi/daq/rest/response/ResponseStreamWriter.java @@ -39,6 +39,8 @@ public class ResponseStreamWriter { @Autowired private ObjectMapper mapper; + @Autowired + private Set defaultResponseFields; /** * Responding with the the contents of the stream by writing into the output stream of the * {@link ServletResponse}. @@ -51,6 +53,9 @@ public class ResponseStreamWriter { public void respond(Stream stream, AbstractQuery query, ServletResponse response) throws IOException { Set includedFields = query.getFields(); + if (includedFields == null) { + includedFields = defaultResponseFields; + } if (query.getAggregations() != null) { includedFields.addAll(query.getAggregations() diff --git a/src/main/resources/rest.properties b/src/main/resources/rest.properties index 4e97ccf..2b5f02a 100644 --- a/src/main/resources/rest.properties +++ b/src/main/resources/rest.properties @@ -1,2 +1,6 @@ # port for the Spring boot application's embedded Tomcat server server.port=8080 + +# defines the fields that are included in the response +# if no fields have been specified by the user +rest.default.response.fields=channel,pulseId,globalMillis,globalNanos,value \ No newline at end of file From 091824a7961971facc0b924ac17d356fc5667d0f Mon Sep 17 00:00:00 2001 From: Zellweger Christof Ralf Date: Tue, 7 Jul 2015 10:53:35 +0200 Subject: [PATCH 7/7] ATEST-99: - improving version for dependency management --- build.gradle | 49 +++++++++++++++++++------------------------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/build.gradle b/build.gradle index e7c9c6b..346adb0 100644 --- a/build.gradle +++ b/build.gradle @@ -1,29 +1,21 @@ version = '1.0.0' buildscript { - dependencies { - classpath("org.springframework.boot:spring-boot-gradle-plugin:1.2.4.RELEASE") - } - repositories { - jcenter() - } + dependencies { classpath(libraries.spring_boot_gradle_plugin) } + repositories { jcenter() } } apply plugin: 'spring-boot' -repositories { - jcenter() -} +repositories { jcenter() } -springBoot { - // when using spring loaded turn on noverify - noverify = true -} +springBoot { // when using spring loaded turn on noverify + noverify = true } applicationDefaultJvmArgs = [ - "-Dfile.encoding=UTF-8", - // if you need to debug java agents: - "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006" + "-Dfile.encoding=UTF-8", + // if you need to debug java agents: + "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006" ] //configurations { @@ -31,24 +23,21 @@ applicationDefaultJvmArgs = [ //} dependencies { - compile (project(':ch.psi.daq.hazelcast')) - compile 'org.springframework.boot:spring-boot-starter-web:1.2.4.RELEASE' - compile 'com.google.code.gson:gson:2+' - compile 'org.apache.commons:commons-lang3:3.4' - - testCompile 'org.springframework.boot:spring-boot-starter-test:1.2.4.RELEASE' - testCompile 'org.skyscreamer:jsonassert:1.2.3' - testCompile 'com.jayway.jsonpath:json-path:2.0.0' + compile (project(':ch.psi.daq.hazelcast')) + compile libraries.spring_boot_starter_web + compile libraries.commons_lang + + testCompile libraries.spring_boot_starter_test + testCompile libraries.jsonassert + testCompile libraries.jsonpath } uploadArchives { - repositories { - mavenDeployer { - pom.artifactId = 'rest' - } - } + repositories { + mavenDeployer { pom.artifactId = 'rest' } + } } task dropItQueryREST(dependsOn: build) << { - exec{ executable "curl"; args "-X", "POST", "-F", "file=@build/libs/ch.psi.daq.rest-" + version + ".jar", "http://dropit.psi.ch:8080"; } + exec{ executable "curl"; args "-X", "POST", "-F", "file=@build/libs/ch.psi.daq.rest-" + version + ".jar", "http://dropit.psi.ch:8080"; } } \ No newline at end of file