diff --git a/Readme.md b/Readme.md index 3594c55..543e57d 100644 --- a/Readme.md +++ b/Readme.md @@ -31,36 +31,24 @@ It is possible to overwrite properties by defining new values in `${HOME}/.confi ### Request ``` -POST http://:/channels +GET http://:/channels + +or + +GET http://:/channels/{regex} ``` ### Example ``` -curl -H "Content-Type: application/json" -X POST -d '{"dbMode":"databuffer"}' http://sf-nube-14.psi.ch:8080/channels +curl -H "Content-Type: application/json" -X GET http://sf-nube-14.psi.ch:8080/channels or -curl -H "Content-Type: application/json" -X POST -d '{"dbMode":"archiverappliance","regex":"TRFCA|TRFCB"}' http://sf-nube-14.psi.ch:8080/channels +curl -H "Content-Type: application/json" -X GET http://sf-nube-14.psi.ch:8080/channels/TRFCB ``` -### Response example - -The response is a JSON array of channel names. - -``` -["channel1","channel2"] -``` - -### Channels Request - -Requests are defined using JSON. -There exist following fields: - -- **dbMode**: Specifies the database to query (values: **databuffer**|archiverappliance). -- **regex**: Specifies a filter (default is **no regex**). Filtering is done using regular expressions (see: [Pattern](https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html). - ## Query Data @@ -68,7 +56,7 @@ There exist following fields: ### Request ``` -POST http://:/query +GET http://:/query ``` ### Example @@ -76,7 +64,7 @@ POST http://:/query A request is performed using JSON. The JSON query defines the channels to be queried, the range, and how the data should be aggregated (this is optional but highly recommended). ``` -curl -H "Content-Type: application/json" -X POST -d '{"channels":["channel1","channel2"],"startPulseId":0,"endPulseId":4}' http://sf-nube-14.psi.ch:8080/channels +curl -H "Content-Type: application/json" -X GET -d '{"channels":["channel1","channel2"],"startPulseId":0,"endPulseId":4}' http://sf-nube-14.psi.ch:8080/channels ``` ### Response example @@ -138,7 +126,7 @@ The response is in JSON. ] ``` -### Query Request +### JSON Query Queries are defined using JSON. There exist following fields: @@ -153,10 +141,9 @@ There exist following fields: - **binSize**: Activates data binning. Specifies the number of pulses per bin for pulse-range queries or the number of milliseconds per bin for time-range queries. - **aggregations**: Activates data aggregation. Array of requested aggregations (see [here](https://github.psi.ch/projects/ST/repos/ch.psi.daq.query/browse/src/main/java/ch/psi/daq/query/model/Aggregation.java) for possible values). These values will be added to the *data* array response. - **aggregationType**: Specifies the type of aggregation (see [here](https://github.psi.ch/projects/ST/repos/ch.psi.daq.query/browse/src/main/java/ch/psi/daq/query/model/AggregationType.java)). The default type is *value* aggregation (e.g., sum([1,2,3])=6). Alternatively, it is possible to define *index* aggregation for multiple arrays in combination with binning (e.g., sum([1,2,3], [3,2,1]) = [4,4,4]). -- **aggregateChannels**: Specifies whether the data of the requested channels should be combined together using the defined aggregation (values: **false**|true) -- **dbMode**: Specifies the database to query (values: **databuffer**|archiverappliance). +- **aggregateChannels**: Specifies whether the data of the requested channels should be combined together using the defined aggregation (values: true|**false**) -### Query Examples +### Example JSON Queries **TODO:** \ No newline at end of file diff --git a/build.gradle b/build.gradle index 1e81a95..c46ec1f 100644 --- a/build.gradle +++ b/build.gradle @@ -24,7 +24,11 @@ applicationDefaultJvmArgs = [ dependencies { compile (project(':ch.psi.daq.query')) - compile libraries.spring_boot_starter_web + compile 'org.hibernate:hibernate-validator:5.2.0.Final' + + compile(libraries.spring_boot_starter_web) { + exclude group: 'org.slf4j', module: 'log4j-over-slf4j' + } compile libraries.commons_lang testCompile libraries.spring_boot_starter_test @@ -39,5 +43,5 @@ uploadArchives { } task dropItQueryREST(dependsOn: build) << { - exec{ executable "curl"; args "-X", "POST", "-F", "file=@build/libs/ch.psi.daq.queryrest-" + version + ".jar", "http://dropit.psi.ch:8080"; } + exec{ executable "curl"; args "-X", "POST", "-F", "file=@build/libs/ch.psi.daq.queryrest-" + version + ".jar", "http://dropit.psi.ch:8080/upload"; } } \ No newline at end of file diff --git a/src/main/java/ch/psi/daq/queryrest/config/QueryRestConfig.java b/src/main/java/ch/psi/daq/queryrest/config/QueryRestConfig.java index 3adb70e..67e8622 100644 --- a/src/main/java/ch/psi/daq/queryrest/config/QueryRestConfig.java +++ b/src/main/java/ch/psi/daq/queryrest/config/QueryRestConfig.java @@ -2,6 +2,7 @@ package ch.psi.daq.queryrest.config; import java.util.EnumMap; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; import java.util.function.Function; @@ -14,7 +15,11 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.PropertySource; import org.springframework.core.env.Environment; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; import org.springframework.util.StringUtils; +import org.springframework.validation.Validator; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.core.JsonFactory; @@ -26,13 +31,14 @@ import ch.psi.daq.cassandra.util.test.CassandraDataGen; import ch.psi.daq.common.json.deserialize.AttributeBasedDeserializer; import ch.psi.daq.common.statistic.StorelessStatistics; import ch.psi.daq.domain.DataEvent; -import ch.psi.daq.query.analyzer.QueryAnalyzerImpl; import ch.psi.daq.query.analyzer.QueryAnalyzer; +import ch.psi.daq.query.analyzer.QueryAnalyzerImpl; import ch.psi.daq.query.config.QueryConfig; import ch.psi.daq.query.model.AbstractQuery; import ch.psi.daq.query.model.Aggregation; import ch.psi.daq.query.model.Query; import ch.psi.daq.query.model.QueryField; +import ch.psi.daq.queryrest.controller.QueryValidator; import ch.psi.daq.queryrest.model.PropertyFilterMixin; import ch.psi.daq.queryrest.response.JsonStreamSerializer; import ch.psi.daq.queryrest.response.ResponseStreamWriter; @@ -40,7 +46,7 @@ import ch.psi.daq.queryrest.response.ResponseStreamWriter; @Configuration @PropertySource(value = {"classpath:queryrest.properties"}) @PropertySource(value = {"file:${user.home}/.config/daq/queryrest.properties"}, ignoreResourceNotFound = true) -public class QueryRestConfig { +public class QueryRestConfig extends WebMvcConfigurerAdapter { private static final String QUERYREST_DEFAULT_RESPONSE_AGGREGATIONS = "queryrest.default.response.aggregations"; @@ -63,6 +69,21 @@ public class QueryRestConfig { @Resource private Environment env; + /** + * {@inheritDoc} + */ + @Override + public void configureMessageConverters(List> converters) { + final MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter(); + /** + * This is necessary so that the message conversion uses the configured object mapper. + * Otherwise, a separate object mapper is instantiated for Springs message conversion. + */ + converter.setObjectMapper(objectMapper()); + converters.add(converter); + super.configureMessageConverters(converters); + } + @Bean public ObjectMapper objectMapper() { ObjectMapper mapper = new ObjectMapper(); @@ -90,7 +111,7 @@ public class QueryRestConfig { return mapper; } - + @Bean public JsonFactory jsonFactory() { return new JsonFactory(); @@ -140,6 +161,11 @@ public class QueryRestConfig { return defaultResponseAggregations; } + + @Bean + public Validator queryValidator() { + return new QueryValidator(); + } // ========================================================================================== // TODO: This is simply for initial / rudimentary testing - remove once further evolved diff --git a/src/main/java/ch/psi/daq/queryrest/controller/QueryRestController.java b/src/main/java/ch/psi/daq/queryrest/controller/QueryRestController.java index 17edf41..3cde28d 100644 --- a/src/main/java/ch/psi/daq/queryrest/controller/QueryRestController.java +++ b/src/main/java/ch/psi/daq/queryrest/controller/QueryRestController.java @@ -2,8 +2,6 @@ package ch.psi.daq.queryrest.controller; import java.io.IOException; import java.util.Collection; -import java.util.LinkedHashSet; -import java.util.LinkedList; import java.util.Map.Entry; import java.util.Set; import java.util.function.Function; @@ -11,10 +9,14 @@ import java.util.stream.Stream; import javax.annotation.Resource; import javax.servlet.http.HttpServletResponse; +import javax.validation.Valid; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; +import org.springframework.validation.Validator; +import org.springframework.web.bind.WebDataBinder; +import org.springframework.web.bind.annotation.InitBinder; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -42,6 +44,9 @@ public class QueryRestController { public static final String CHANNELS = "channels"; public static final String QUERY = "query"; + @Resource + private Validator queryValidator; + @Resource private ResponseStreamWriter responseStreamWriter; @@ -60,6 +65,15 @@ public class QueryRestController { @Resource(name = QueryRestConfig.BEAN_NAME_DEFAULT_RESPONSE_AGGREGATIONS) private Set defaultResponseAggregations; + + @InitBinder + protected void initBinder(WebDataBinder binder) { + /* + * This allows to use the @Valid annotation in the methods below. + */ + binder.addValidators(queryValidator); + } + @RequestMapping( value = CHANNELS, method = {RequestMethod.GET, RequestMethod.POST}, @@ -82,20 +96,16 @@ public class QueryRestController { @RequestMapping( value = QUERY, method = RequestMethod.POST, - consumes = {MediaType.APPLICATION_JSON_VALUE}, - produces = {MediaType.APPLICATION_JSON_VALUE}) - public void executeQuery(@RequestBody AbstractQuery query, HttpServletResponse res) throws IOException { + consumes = {MediaType.APPLICATION_JSON_VALUE}) + public void executeQuery(@RequestBody @Valid AbstractQuery query, HttpServletResponse res) throws IOException { try { - LOGGER.debug("Execute query '{}'", query.getClass().getSimpleName()); + LOGGER.debug("Executing query '{}'", query.getClass().getSimpleName()); QueryAnalyzer queryAnalizer = queryAnalizerFactory.apply(query); queryAnalizer.validate(); - extendQuery(query); - // all the magic happens here - Stream>> channelToDataEvents = - getQueryProcessor(query.getDBMode()).process(queryAnalizer); + Stream>> channelToDataEvents = getQueryProcessor(query.getDBMode()).process(queryAnalizer); // do post-process Stream> channelToData = queryAnalizer.postProcess(channelToDataEvents); @@ -119,15 +129,6 @@ public class QueryRestController { } } - private void extendQuery(AbstractQuery query) { - if (query.getFields() == null || query.getFields().isEmpty()) { - query.setFields(new LinkedHashSet<>(defaultResponseFields)); - } - if (query.getAggregations() == null || query.getAggregations().isEmpty()) { - query.setAggregations(new LinkedList<>(defaultResponseAggregations)); - } - } - // ========================================================================================== // TODO: This is simply for initial / rudimentary testing - remove once further evolved diff --git a/src/main/java/ch/psi/daq/queryrest/controller/QueryValidator.java b/src/main/java/ch/psi/daq/queryrest/controller/QueryValidator.java new file mode 100644 index 0000000..8bc1705 --- /dev/null +++ b/src/main/java/ch/psi/daq/queryrest/controller/QueryValidator.java @@ -0,0 +1,57 @@ +/** + * + */ +package ch.psi.daq.queryrest.controller; + +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.Set; + +import javax.annotation.Resource; + +import org.springframework.validation.Errors; +import org.springframework.validation.Validator; + +import ch.psi.daq.query.model.AbstractQuery; +import ch.psi.daq.query.model.Aggregation; +import ch.psi.daq.query.model.QueryField; +import ch.psi.daq.queryrest.config.QueryRestConfig; + +/** + * @author zellweger_c + * + */ +public class QueryValidator implements Validator { + + @Resource(name = QueryRestConfig.BEAN_NAME_DEFAULT_RESPONSE_FIELDS) + private Set defaultResponseFields; + + @Resource(name = QueryRestConfig.BEAN_NAME_DEFAULT_RESPONSE_AGGREGATIONS) + private Set defaultResponseAggregations; + + /** + * {@inheritDoc} + */ + @Override + public boolean supports(Class clazz) { + return AbstractQuery.class.isAssignableFrom(clazz); + } + + /** + * {@inheritDoc} + */ + @Override + public void validate(Object target, Errors errors) { + + AbstractQuery query = (AbstractQuery) target; + + if (query.getFields() == null || query.getFields().isEmpty()) { + query.setFields(new LinkedHashSet<>(defaultResponseFields)); + } + + if (query.getAggregations() == null || query.getAggregations().isEmpty()) { + query.setAggregations(new LinkedList<>(defaultResponseAggregations)); + } + } + +} diff --git a/src/main/resources/queryrest.properties b/src/main/resources/queryrest.properties index db51a1b..1050546 100644 --- a/src/main/resources/queryrest.properties +++ b/src/main/resources/queryrest.properties @@ -4,4 +4,4 @@ server.port=8080 # defines the fields that are included in the response # if no fields have been specified by the user queryrest.default.response.fields=channel,pulseId,globalMillis,globalNanos,value -queryrest.default.response.aggregations=min,max,sum +queryrest.default.response.aggregations=min,max,sum \ No newline at end of file diff --git a/src/test/java/ch/psi/daq/test/queryrest/AbstractDaqRestTest.java b/src/test/java/ch/psi/daq/test/queryrest/AbstractDaqRestTest.java index 5013654..82e77db 100644 --- a/src/test/java/ch/psi/daq/test/queryrest/AbstractDaqRestTest.java +++ b/src/test/java/ch/psi/daq/test/queryrest/AbstractDaqRestTest.java @@ -5,9 +5,10 @@ import javax.annotation.Resource; import org.junit.Before; import org.junit.runner.RunWith; import org.mockito.MockitoAnnotations; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.SpringApplicationConfiguration; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.annotation.DirtiesContext.ClassMode; -import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestExecutionListeners; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; @@ -18,21 +19,25 @@ import org.springframework.web.context.WebApplicationContext; import com.fasterxml.jackson.databind.ObjectMapper; +import ch.psi.daq.queryrest.QueryRestApplication; +import ch.psi.daq.queryrest.config.QueryRestConfig; import ch.psi.daq.test.cassandra.CassandraDaqUnitDependencyInjectionTestExecutionListener; @TestExecutionListeners({ CassandraDaqUnitDependencyInjectionTestExecutionListener.class, DependencyInjectionTestExecutionListener.class}) -//@SpringApplicationConfiguration(classes = {QueryRestApplication.class, DaqWebMvcConfig.class}) -//@EmbeddedCassandra +@SpringApplicationConfiguration(classes = { + QueryRestApplication.class + ,QueryRestConfig.class + ,DaqWebMvcConfig.class +}) @DirtiesContext(classMode = ClassMode.AFTER_EACH_TEST_METHOD) @WebAppConfiguration -@ContextConfiguration(classes = DaqWebMvcConfig.class) @RunWith(SpringJUnit4ClassRunner.class) public abstract class AbstractDaqRestTest { - @Resource + @Autowired protected WebApplicationContext webApplicationContext; protected MockMvc mockMvc; diff --git a/src/test/java/ch/psi/daq/test/queryrest/DaqWebMvcConfig.java b/src/test/java/ch/psi/daq/test/queryrest/DaqWebMvcConfig.java index baf858f..00c82ef 100644 --- a/src/test/java/ch/psi/daq/test/queryrest/DaqWebMvcConfig.java +++ b/src/test/java/ch/psi/daq/test/queryrest/DaqWebMvcConfig.java @@ -1,6 +1,7 @@ package ch.psi.daq.test.queryrest; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.PropertySource; @@ -8,16 +9,20 @@ import org.springframework.context.annotation.PropertySources; import org.springframework.web.servlet.config.annotation.EnableWebMvc; import org.springframework.web.servlet.config.annotation.WebMvcConfigurationSupport; -import ch.psi.daq.query.processor.QueryProcessorLocal; +import ch.psi.daq.domain.reader.DataReader; import ch.psi.daq.query.processor.QueryProcessor; +import ch.psi.daq.query.processor.QueryProcessorLocal; +import ch.psi.daq.test.cassandra.admin.CassandraAdmin; +import ch.psi.daq.test.cassandra.admin.CassandraAdminImpl; import ch.psi.daq.test.query.config.LocalQueryTestConfig; import ch.psi.daq.test.queryrest.query.DummyDataReader; @Configuration +@ComponentScan +@EnableWebMvc @PropertySources(value = { @PropertySource(value = {"classpath:queryrest-test.properties"}) }) -@EnableWebMvc public class DaqWebMvcConfig extends WebMvcConfigurationSupport { // ensure that properties in dispatcher.properties are loaded first and then overwritten by the @@ -25,9 +30,19 @@ public class DaqWebMvcConfig extends WebMvcConfigurationSupport { @Import(value = {LocalQueryTestConfig.class}) static class InnerConfiguration { } - + @Bean public QueryProcessor queryProcessor() { return new QueryProcessorLocal(new DummyDataReader()); } -} + + @Bean + public DataReader dataReader() { + return new DummyDataReader(); + } + + @Bean + public CassandraAdmin cassandraAdmin() { + return new CassandraAdminImpl(); + } +} \ No newline at end of file diff --git a/src/test/java/ch/psi/daq/test/queryrest/controller/DaqRestControllerTest.java b/src/test/java/ch/psi/daq/test/queryrest/controller/DaqRestControllerTest.java index ac061eb..269cef1 100644 --- a/src/test/java/ch/psi/daq/test/queryrest/controller/DaqRestControllerTest.java +++ b/src/test/java/ch/psi/daq/test/queryrest/controller/DaqRestControllerTest.java @@ -48,6 +48,23 @@ public class DaqRestControllerTest extends AbstractDaqRestTest { @After public void tearDown() throws Exception {} + @Test + public void testChannelNameQuery() throws Exception { + + this.mockMvc.perform( + MockMvcRequestBuilders + .get("/" + QueryRestController.CHANNELS) + .contentType(MediaType.APPLICATION_JSON)) + .andDo(MockMvcResultHandlers.print()) + .andExpect(MockMvcResultMatchers.status().isOk()) + .andExpect(MockMvcResultMatchers.jsonPath("$").isArray()) + .andExpect(MockMvcResultMatchers.jsonPath("$[0]").exists()) + .andExpect(MockMvcResultMatchers.jsonPath("$[0]").value("testChannel1")) + .andExpect(MockMvcResultMatchers.jsonPath("$[1]").exists()) + .andExpect(MockMvcResultMatchers.jsonPath("$[1]").value("testChannel2")) + .andExpect(MockMvcResultMatchers.jsonPath("$[2]").doesNotExist()); + } + @Test public void testPulseRangeQuery() throws Exception { PulseRangeQuery request = new PulseRangeQuery( @@ -56,9 +73,13 @@ public class DaqRestControllerTest extends AbstractDaqRestTest { TEST_CHANNEL_NAMES); String content = mapper.writeValueAsString(request); - + System.out.println(content); + + content = "{\"channels\":[\"testChannel1\",\"testChannel2\"],\"startPulseId\":100,\"endPulseId\":101}"; + this.mockMvc - .perform(MockMvcRequestBuilders.post(QueryRestController.QUERY) + .perform(MockMvcRequestBuilders + .post("/" + QueryRestController.QUERY) .contentType(MediaType.APPLICATION_JSON) .content(content)) @@ -86,8 +107,8 @@ public class DaqRestControllerTest extends AbstractDaqRestTest { String content = mapper.writeValueAsString(request); - this.mockMvc - .perform(MockMvcRequestBuilders.post(QueryRestController.QUERY) + this.mockMvc.perform(MockMvcRequestBuilders + .post("/" + QueryRestController.QUERY) .contentType(MediaType.APPLICATION_JSON) .content(content)) @@ -116,12 +137,15 @@ public class DaqRestControllerTest extends AbstractDaqRestTest { TEST_CHANNEL_NAMES); String content = mapper.writeValueAsString(request); + System.out.println(content); this.mockMvc - .perform(MockMvcRequestBuilders.post(QueryRestController.QUERY) + .perform( + MockMvcRequestBuilders + .post("/" + QueryRestController.QUERY) .contentType(MediaType.APPLICATION_JSON) - .content(content)) - + .content(content) + ) .andDo(MockMvcResultHandlers.print()) .andExpect(MockMvcResultMatchers.status().isOk()) .andExpect(MockMvcResultMatchers.jsonPath("$").isArray()) @@ -150,7 +174,7 @@ public class DaqRestControllerTest extends AbstractDaqRestTest { String content = mapper.writeValueAsString(request); this.mockMvc - .perform(MockMvcRequestBuilders.post(QueryRestController.QUERY) + .perform(MockMvcRequestBuilders.post("/" + QueryRestController.QUERY) .contentType(MediaType.APPLICATION_JSON) .content(content)) @@ -177,20 +201,4 @@ public class DaqRestControllerTest extends AbstractDaqRestTest { .andExpect(MockMvcResultMatchers.jsonPath("$.minima.max.event.pulseId").exists()) .andExpect(MockMvcResultMatchers.jsonPath("$.minima.max.event.pulseId").value(101)); } - - @Test - public void testChannelNameQuery() throws Exception { - this.mockMvc - .perform(MockMvcRequestBuilders.get(QueryRestController.CHANNELS) - .contentType(MediaType.APPLICATION_JSON)) - - .andDo(MockMvcResultHandlers.print()) - .andExpect(MockMvcResultMatchers.status().isOk()) - .andExpect(MockMvcResultMatchers.jsonPath("$").isArray()) - .andExpect(MockMvcResultMatchers.jsonPath("$[0]").exists()) - .andExpect(MockMvcResultMatchers.jsonPath("$[0]").value("testChannel1")) - .andExpect(MockMvcResultMatchers.jsonPath("$[1]").exists()) - .andExpect(MockMvcResultMatchers.jsonPath("$[1]").value("testChannel2")) - .andExpect(MockMvcResultMatchers.jsonPath("$[2]").doesNotExist()); - } } diff --git a/src/test/java/ch/psi/daq/test/queryrest/query/DummyDataReader.java b/src/test/java/ch/psi/daq/test/queryrest/query/DummyDataReader.java index 2dcf153..e55cf11 100644 --- a/src/test/java/ch/psi/daq/test/queryrest/query/DummyDataReader.java +++ b/src/test/java/ch/psi/daq/test/queryrest/query/DummyDataReader.java @@ -1,7 +1,6 @@ package ch.psi.daq.test.queryrest.query; import java.util.List; -import java.util.UUID; import java.util.regex.Pattern; import java.util.stream.LongStream; import java.util.stream.Stream; @@ -52,7 +51,8 @@ public class DummyDataReader implements DataReader { i, i, 0, - "data_" + UUID.randomUUID().toString()); + 123 // dummy value + ); }); } }