ITQC/gls_docs
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into ATEST-128

Browse Source
This commit is contained in:
Fabian Märki
2015-07-31 14:36:26 +02:00
parent 47d921e1a4 83613f883f
commit 46c1406d57
10 changed files with 216 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/main/java/ch/psi/daq/queryrest/config/QueryRestConfig.java
View File

@ -2,6 +2,7 @@ package ch.psi.daq.queryrest.config;
import java.util.EnumMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
@ -14,7 +15,11 @@ import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.PropertySource;
import org.springframework.core.env.Environment;
import org.springframework.http.converter.HttpMessageConverter;
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
import org.springframework.util.StringUtils;
import org.springframework.validation.Validator;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.core.JsonFactory;
@ -26,13 +31,14 @@ import ch.psi.daq.cassandra.util.test.CassandraDataGen;
import ch.psi.daq.common.json.deserialize.AttributeBasedDeserializer;
import ch.psi.daq.common.statistic.StorelessStatistics;
import ch.psi.daq.domain.DataEvent;
import ch.psi.daq.query.analyzer.QueryAnalyzerImpl;
import ch.psi.daq.query.analyzer.QueryAnalyzer;
import ch.psi.daq.query.analyzer.QueryAnalyzerImpl;
import ch.psi.daq.query.config.QueryConfig;
import ch.psi.daq.query.model.AbstractQuery;
import ch.psi.daq.query.model.Aggregation;
import ch.psi.daq.query.model.Query;
import ch.psi.daq.query.model.QueryField;
import ch.psi.daq.queryrest.controller.QueryValidator;
import ch.psi.daq.queryrest.model.PropertyFilterMixin;
import ch.psi.daq.queryrest.response.JsonStreamSerializer;
import ch.psi.daq.queryrest.response.ResponseStreamWriter;
@ -40,7 +46,7 @@ import ch.psi.daq.queryrest.response.ResponseStreamWriter;
@Configuration
@PropertySource(value = {"classpath:queryrest.properties"})
@PropertySource(value = {"file:${user.home}/.config/daq/queryrest.properties"}, ignoreResourceNotFound = true)
public class QueryRestConfig {
public class QueryRestConfig extends WebMvcConfigurerAdapter {
private static final String QUERYREST_DEFAULT_RESPONSE_AGGREGATIONS = "queryrest.default.response.aggregations";
@ -63,6 +69,21 @@ public class QueryRestConfig {
@Resource
private Environment env;
/**
* {@inheritDoc}
*/
@Override
public void configureMessageConverters(List<HttpMessageConverter<?>> converters) {
final MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter();
/**
* This is necessary so that the message conversion uses the configured object mapper.
* Otherwise, a separate object mapper is instantiated for Springs message conversion.
*/
converter.setObjectMapper(objectMapper());
converters.add(converter);
super.configureMessageConverters(converters);
}
@Bean
public ObjectMapper objectMapper() {
ObjectMapper mapper = new ObjectMapper();
@ -90,7 +111,7 @@ public class QueryRestConfig {
return mapper;
}
@Bean
public JsonFactory jsonFactory() {
return new JsonFactory();
@ -140,6 +161,11 @@ public class QueryRestConfig {
return defaultResponseAggregations;
}
@Bean
public Validator queryValidator() {
return new QueryValidator();
}
// ==========================================================================================
// TODO: This is simply for initial / rudimentary testing - remove once further evolved

40
src/main/java/ch/psi/daq/queryrest/controller/QueryRestController.java
View File

@ -2,8 +2,6 @@ package ch.psi.daq.queryrest.controller;
import java.io.IOException;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.Map.Entry;
import java.util.Set;
import java.util.function.Function;
@ -11,10 +9,14 @@ import java.util.stream.Stream;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletResponse;
import javax.validation.Valid;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.MediaType;
import org.springframework.validation.Validator;
import org.springframework.web.bind.WebDataBinder;
import org.springframework.web.bind.annotation.InitBinder;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@ -42,6 +44,9 @@ public class QueryRestController {
public static final String CHANNELS = "channels";
public static final String QUERY = "query";
@Resource
private Validator queryValidator;
@Resource
private ResponseStreamWriter responseStreamWriter;
@ -60,6 +65,15 @@ public class QueryRestController {
@Resource(name = QueryRestConfig.BEAN_NAME_DEFAULT_RESPONSE_AGGREGATIONS)
private Set<Aggregation> defaultResponseAggregations;
@InitBinder
protected void initBinder(WebDataBinder binder) {
/*
* This allows to use the @Valid annotation in the methods below.
*/
binder.addValidators(queryValidator);
}
@RequestMapping(
value = CHANNELS,
method = {RequestMethod.GET, RequestMethod.POST},
@ -82,20 +96,15 @@ public class QueryRestController {
@RequestMapping(
value = QUERY,
method = RequestMethod.POST,
consumes = {MediaType.APPLICATION_JSON_VALUE},
produces = {MediaType.APPLICATION_JSON_VALUE})
public void executeQuery(@RequestBody AbstractQuery query, HttpServletResponse res) throws IOException {
consumes = {MediaType.APPLICATION_JSON_VALUE})
public void executeQuery(@RequestBody @Valid AbstractQuery query, HttpServletResponse res) throws IOException {
try {
LOGGER.debug("Execute query '{}'", query.getClass().getSimpleName());
LOGGER.debug("Executing query '{}'", query.getClass().getSimpleName());
QueryAnalyzer queryAnalizer = queryAnalizerFactory.apply(query);
queryAnalizer.validate();
extendQuery(query);
// all the magic happens here
Stream<Entry<String, Stream<? extends DataEvent>>> channelToDataEvents =
getQueryProcessor(query.getDBMode()).process(queryAnalizer);
Stream<Entry<String, Stream<? extends DataEvent>>> channelToDataEvents = getQueryProcessor(query.getDBMode()).process(queryAnalizer);
// do post-process
Stream<Entry<String, ?>> channelToData = queryAnalizer.postProcess(channelToDataEvents);
@ -119,15 +128,6 @@ public class QueryRestController {
}
}
private void extendQuery(AbstractQuery query) {
if (query.getFields() == null || query.getFields().isEmpty()) {
query.setFields(new LinkedHashSet<>(defaultResponseFields));
}
if (query.getAggregations() == null || query.getAggregations().isEmpty()) {
query.setAggregations(new LinkedList<>(defaultResponseAggregations));
}
}
// ==========================================================================================
// TODO: This is simply for initial / rudimentary testing - remove once further evolved

85
src/main/java/ch/psi/daq/queryrest/controller/QueryValidator.java Normal file
View File

@ -0,0 +1,85 @@
/**
*
*/
package ch.psi.daq.queryrest.controller;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.Set;
import javax.annotation.Resource;
import org.springframework.validation.Errors;
import org.springframework.validation.Validator;
import ch.psi.daq.query.model.AbstractQuery;
import ch.psi.daq.query.model.AbstractTimeRangeQuery;
import ch.psi.daq.query.model.Aggregation;
import ch.psi.daq.query.model.DBMode;
import ch.psi.daq.query.model.QueryField;
import ch.psi.daq.query.range.QueryRange;
import ch.psi.daq.queryrest.config.QueryRestConfig;
/**
* @author zellweger_c
*
*/
public class QueryValidator implements Validator {
@Resource(name = QueryRestConfig.BEAN_NAME_DEFAULT_RESPONSE_FIELDS)
private Set<QueryField> defaultResponseFields;
@Resource(name = QueryRestConfig.BEAN_NAME_DEFAULT_RESPONSE_AGGREGATIONS)
private Set<Aggregation> defaultResponseAggregations;
/**
* {@inheritDoc}
*/
@Override
public boolean supports(Class<?> clazz) {
return AbstractQuery.class.isAssignableFrom(clazz);
}
/**
* {@inheritDoc}
*/
@Override
public void validate(Object target, Errors errors) {
AbstractQuery query = (AbstractQuery) target;
QueryRange queryRange = query.getQueryRange();
if (!queryRange.isPulseIdRangeDefined() && !queryRange.isTimeRangeDefined()) {
errors.reject("", "Either time or pulseId range must be defined by the query!");
}
if (queryRange.isPulseIdRangeDefined()) {
if (queryRange.getStartPulseId() > queryRange.getEndPulseId()) {
errors.rejectValue("pulseId", "pulseId", "Start pulse-id must be smaller or equals end pulse-id!");
}
} else if (queryRange.isTimeRangeDefined()) {
if (queryRange.getStartMillis() > queryRange.getEndMillis()) {
errors.rejectValue("millis", "millis", "Start millis must be smaller or equals to end millis!");
} else if (queryRange.getStartMillis() == queryRange.getEndMillis()
&& queryRange.getStartNanos() > queryRange.getEndNanos()) {
errors.rejectValue("nanos", "nanos", "Start nanos must be smaller or equals to end nanos!");
}
}
if (DBMode.archiverappliance.equals(query.getDBMode())) {
if (!(query instanceof AbstractTimeRangeQuery)) {
errors.reject("", "ArchiverAppliance supports time range queries only!");
}
}
if (query.getFields() == null || query.getFields().isEmpty()) {
query.setFields(new LinkedHashSet<>(defaultResponseFields));
}
if (query.getAggregations() == null || query.getAggregations().isEmpty()) {
query.setAggregations(new LinkedList<>(defaultResponseAggregations));
}
}
}

2
src/main/resources/queryrest.properties
View File

@ -4,4 +4,4 @@ server.port=8080
# defines the fields that are included in the response
# if no fields have been specified by the user
queryrest.default.response.fields=channel,pulseId,globalMillis,globalNanos,value
queryrest.default.response.aggregations=min,max,sum
queryrest.default.response.aggregations=min,max,sum

15
src/test/java/ch/psi/daq/test/queryrest/AbstractDaqRestTest.java
View File

@ -5,9 +5,10 @@ import javax.annotation.Resource;
import org.junit.Before;
import org.junit.runner.RunWith;
import org.mockito.MockitoAnnotations;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.SpringApplicationConfiguration;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.annotation.DirtiesContext.ClassMode;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.TestExecutionListeners;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.support.DependencyInjectionTestExecutionListener;
@ -18,21 +19,25 @@ import org.springframework.web.context.WebApplicationContext;
import com.fasterxml.jackson.databind.ObjectMapper;
import ch.psi.daq.queryrest.QueryRestApplication;
import ch.psi.daq.queryrest.config.QueryRestConfig;
import ch.psi.daq.test.cassandra.CassandraDaqUnitDependencyInjectionTestExecutionListener;
@TestExecutionListeners({
CassandraDaqUnitDependencyInjectionTestExecutionListener.class,
DependencyInjectionTestExecutionListener.class})
//@SpringApplicationConfiguration(classes = {QueryRestApplication.class, DaqWebMvcConfig.class})
//@EmbeddedCassandra
@SpringApplicationConfiguration(classes = {
QueryRestApplication.class
,QueryRestConfig.class
,DaqWebMvcConfig.class
})
@DirtiesContext(classMode = ClassMode.AFTER_EACH_TEST_METHOD)
@WebAppConfiguration
@ContextConfiguration(classes = DaqWebMvcConfig.class)
@RunWith(SpringJUnit4ClassRunner.class)
public abstract class AbstractDaqRestTest {
@Resource
@Autowired
protected WebApplicationContext webApplicationContext;
protected MockMvc mockMvc;

23
src/test/java/ch/psi/daq/test/queryrest/DaqWebMvcConfig.java
View File

@ -1,6 +1,7 @@
package ch.psi.daq.test.queryrest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.PropertySource;
@ -8,16 +9,20 @@ import org.springframework.context.annotation.PropertySources;
import org.springframework.web.servlet.config.annotation.EnableWebMvc;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurationSupport;
import ch.psi.daq.query.processor.QueryProcessorLocal;
import ch.psi.daq.domain.reader.DataReader;
import ch.psi.daq.query.processor.QueryProcessor;
import ch.psi.daq.query.processor.QueryProcessorLocal;
import ch.psi.daq.test.cassandra.admin.CassandraAdmin;
import ch.psi.daq.test.cassandra.admin.CassandraAdminImpl;
import ch.psi.daq.test.query.config.LocalQueryTestConfig;
import ch.psi.daq.test.queryrest.query.DummyDataReader;
@Configuration
@ComponentScan
@EnableWebMvc
@PropertySources(value = {
@PropertySource(value = {"classpath:queryrest-test.properties"})
})
@EnableWebMvc
public class DaqWebMvcConfig extends WebMvcConfigurationSupport {
// ensure that properties in dispatcher.properties are loaded first and then overwritten by the
@ -25,9 +30,19 @@ public class DaqWebMvcConfig extends WebMvcConfigurationSupport {
@Import(value = {LocalQueryTestConfig.class})
static class InnerConfiguration {
}
@Bean
public QueryProcessor queryProcessor() {
return new QueryProcessorLocal(new DummyDataReader());
}
}
@Bean
public DataReader dataReader() {
return new DummyDataReader();
}
@Bean
public CassandraAdmin cassandraAdmin() {
return new CassandraAdminImpl();
}
}

56
src/test/java/ch/psi/daq/test/queryrest/controller/DaqRestControllerTest.java
View File

@ -48,6 +48,23 @@ public class DaqRestControllerTest extends AbstractDaqRestTest {
@After
public void tearDown() throws Exception {}
@Test
public void testChannelNameQuery() throws Exception {
this.mockMvc.perform(
MockMvcRequestBuilders
.get("/" + QueryRestController.CHANNELS)
.contentType(MediaType.APPLICATION_JSON))
.andDo(MockMvcResultHandlers.print())
.andExpect(MockMvcResultMatchers.status().isOk())
.andExpect(MockMvcResultMatchers.jsonPath("$").isArray())
.andExpect(MockMvcResultMatchers.jsonPath("$[0]").exists())
.andExpect(MockMvcResultMatchers.jsonPath("$[0]").value("testChannel1"))
.andExpect(MockMvcResultMatchers.jsonPath("$[1]").exists())
.andExpect(MockMvcResultMatchers.jsonPath("$[1]").value("testChannel2"))
.andExpect(MockMvcResultMatchers.jsonPath("$[2]").doesNotExist());
}
@Test
public void testPulseRangeQuery() throws Exception {
PulseRangeQuery request = new PulseRangeQuery(
@ -56,9 +73,13 @@ public class DaqRestControllerTest extends AbstractDaqRestTest {
TEST_CHANNEL_NAMES);
String content = mapper.writeValueAsString(request);
System.out.println(content);
content = "{\"channels\":[\"testChannel1\",\"testChannel2\"],\"startPulseId\":100,\"endPulseId\":101}";
this.mockMvc
.perform(MockMvcRequestBuilders.post(QueryRestController.QUERY)
.perform(MockMvcRequestBuilders
.post("/" + QueryRestController.QUERY)
.contentType(MediaType.APPLICATION_JSON)
.content(content))
@ -86,8 +107,8 @@ public class DaqRestControllerTest extends AbstractDaqRestTest {
String content = mapper.writeValueAsString(request);
this.mockMvc
.perform(MockMvcRequestBuilders.post(QueryRestController.QUERY)
this.mockMvc.perform(MockMvcRequestBuilders
.post("/" + QueryRestController.QUERY)
.contentType(MediaType.APPLICATION_JSON)
.content(content))
@ -116,12 +137,15 @@ public class DaqRestControllerTest extends AbstractDaqRestTest {
TEST_CHANNEL_NAMES);
String content = mapper.writeValueAsString(request);
System.out.println(content);
this.mockMvc
.perform(MockMvcRequestBuilders.post(QueryRestController.QUERY)
.perform(
MockMvcRequestBuilders
.post("/" + QueryRestController.QUERY)
.contentType(MediaType.APPLICATION_JSON)
.content(content))
.content(content)
)
.andDo(MockMvcResultHandlers.print())
.andExpect(MockMvcResultMatchers.status().isOk())
.andExpect(MockMvcResultMatchers.jsonPath("$").isArray())
@ -150,7 +174,7 @@ public class DaqRestControllerTest extends AbstractDaqRestTest {
String content = mapper.writeValueAsString(request);
this.mockMvc
.perform(MockMvcRequestBuilders.post(QueryRestController.QUERY)
.perform(MockMvcRequestBuilders.post("/" + QueryRestController.QUERY)
.contentType(MediaType.APPLICATION_JSON)
.content(content))
@ -177,20 +201,4 @@ public class DaqRestControllerTest extends AbstractDaqRestTest {
.andExpect(MockMvcResultMatchers.jsonPath("$.minima.max.event.pulseId").exists())
.andExpect(MockMvcResultMatchers.jsonPath("$.minima.max.event.pulseId").value(101));
}
@Test
public void testChannelNameQuery() throws Exception {
this.mockMvc
.perform(MockMvcRequestBuilders.get(QueryRestController.CHANNELS)
.contentType(MediaType.APPLICATION_JSON))
.andDo(MockMvcResultHandlers.print())
.andExpect(MockMvcResultMatchers.status().isOk())
.andExpect(MockMvcResultMatchers.jsonPath("$").isArray())
.andExpect(MockMvcResultMatchers.jsonPath("$[0]").exists())
.andExpect(MockMvcResultMatchers.jsonPath("$[0]").value("testChannel1"))
.andExpect(MockMvcResultMatchers.jsonPath("$[1]").exists())
.andExpect(MockMvcResultMatchers.jsonPath("$[1]").value("testChannel2"))
.andExpect(MockMvcResultMatchers.jsonPath("$[2]").doesNotExist());
}
}

4
src/test/java/ch/psi/daq/test/queryrest/query/DummyDataReader.java
View File

@ -1,7 +1,6 @@
package ch.psi.daq.test.queryrest.query;
import java.util.List;
import java.util.UUID;
import java.util.regex.Pattern;
import java.util.stream.LongStream;
import java.util.stream.Stream;
@ -52,7 +51,8 @@ public class DummyDataReader implements DataReader {
i,
i,
0,
"data_" + UUID.randomUUID().toString());
123 // dummy value
);
});
}
}