diff --git a/core-buffer/include/buffer_config.hpp b/core-buffer/include/buffer_config.hpp
index 1f9b812..b9a8349 100644
--- a/core-buffer/include/buffer_config.hpp
+++ b/core-buffer/include/buffer_config.hpp
@@ -45,6 +45,8 @@ namespace core_buffer {
// If the RB is empty, how much time to wait before trying to read it again.
const size_t RB_READ_RETRY_INTERVAL_MS = 5;
+
+ const size_t LIVE_READ_BLOCK_SIZE = 10;
}
#endif //BUFFERCONFIG_HPP
diff --git a/core-writer/CMakeLists.txt b/core-writer/CMakeLists.txt
deleted file mode 100644
index 02f13f8..0000000
--- a/core-writer/CMakeLists.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-add_subdirectory(external/)
-
-file(GLOB SOURCES
- src/*.cpp
- src/writer/*.cpp
- src/receiver/*.cpp
- src/module/*.cpp)
-
-add_library(core-writer STATIC ${SOURCES})
-include_directories(core-writer external/)
-
-target_include_directories(core-writer PUBLIC include/)
-target_include_directories(core-writer PUBLIC external/)
-
-if(CMAKE_BUILD_TYPE STREQUAL "Debug")
- target_compile_definitions(core-writer PRIVATE DEBUG_OUTPUT)
-endif()
-
-enable_testing()
-add_subdirectory(test/)
\ No newline at end of file
diff --git a/core-writer/docs/diagrams.drawio b/core-writer/docs/diagrams.drawio
deleted file mode 100644
index fc22e7d..0000000
--- a/core-writer/docs/diagrams.drawio
+++ /dev/null
@@ -1 +0,0 @@
-7Vxbd6I6FP41fWwWIYTAY9u532d6zsyZ8zILJSozaCzEVs+vP4mAEhIBL1h7KWtRs4kBvn3fSTxDV+P56ySYjj6ykMZnthXOz9CLM9smmIizJCwyArShn1GGSRTmtDXhOvqP5kQrp86ikKZKR85YzKOpSuyzyYT2uUILkoTdqd0GLFbvOg2GVCNc94NYp/6IQj7KqJ5N1vQ3NBqOijtDN3+/cVB0zt8kHQUhuyuR0MszdJUwxrNP4/kVjSV4BS7Z915tuLp6sIROeJsvvPtx8+maw9ep6/bGd/wrQdfJOXSzYW6DeJa/cf60fFFAMEzYbJp3owmncxPwQa/obukPBlevK+SEsjHlyUJ0KQaynOwruYwUI9yt8XYL2qiEte25ADs2sojryzPJ3yXIWT5c3WmNiviQA7MNSPAYINXwpwY6BTmMgWUhz3Ehzs5YQxJiCVrpMOCKIMClUbqC1W9GVYA6CakcxDpDl3ejiNPradCXV++EvRG0ER+Lm76A4uNG9JtRbhJQBWWoyyd0KqgiA6y2Civ2OsIVnTyup4qcruffaJ9Gt3TpdeSDJwIp2RglNAjFh2+XcswJp0nQ5xGbaEgLzLgKZ8oT9odesZglgjJhE9HzchDFcYUUxNFwIpp9KocXBMmBSHimi/zCOApDeRsj/1QOD9iE577VtvZjaYNBQrrJQdhkvK39eYjmP7/zcPgp+D5iLFyMp+/fjwxW5Vs0GV7OBgMBYpU7widP5cf+Io4EXEmzMvQyXD/0VoSg/2e4RPvzjItRaE6Pgx6Nv7A0WoqFgY0fKh04m7bheo9xzsYrHrZkmIHZG3loYx/YpHTgivkjIHO4uduFujm0IHAavAwkNliNIc92RwJh8N7RRIAukBYKcZr62hlvoWcB5JYOUuFtxdIijbeOiDH80uG5BuV2wHoEeSZd8RY3+zo6CS9kIiBZEwdpGvVV/gqIk8U/onFuAWtF+Ck5AYjnFYQX85w3WWtRbn2hSSReSHJ9F/bxIBlS3mzTaKjkKrV+1WRzC1pC44BLr6YkSwbe5Hf4woSHK8nQBpddhO8OAmW9royfslnSp/mQ5XSlchfb8dRhvYrHyFDTBlrK0QqTPURLdyR9Nh5H/NlkqHyBHnDK2u6dtskoLMRBTEbFXOAGY/G47YJDHIA9uD5U1+IjUHYsXoXBbe2CyF9VaVSLB45zXDNht8i2TkScTkZMkBIcqtHlgbwHEjEs9kt/ZJu7dC0zjiYzWTr5tBwL8h1TKaqwF67IQpQE4cT9il5JFYpCg/TJ5Rj1fH1w8UKLMmWjgZ9HfGnfRYKB83Zm4hHx8/baxsvGotSoZhcrd2Gp7gKiI+QqmQGuwYs8yNgF+hC4SlyhGqNquap1FmPZQBFlNakhLvDLRg4d1Q85203+7G4qNk2A7Fym8GxAlBqUZkOIp+Du36MFgUg3GELqr/MmS/iIDdkkiF+uqRVbve7zgcky4dKq/KacL/KqbjDjbIPNUewNrrU2Op9r5aZRd/dUynPkAMtyVwd2FSk4l/NUnk8clJ8LT7OtkgpZQUq5UrkNrlTE/FY6KvxBsCh1m8oO6eZ33eUZ1kKZ3e2w9Q/PEM+kNLl9OPFMvdFrHc+cC7VRfUNFDiFRM1tHN0cnFdJg03y2G0sehtGtwlv3ZibXJywnkM7Tpa25EB2gNZ0vIS+ui09D+T+NRTMfSzzbcrjsyqHuMJ5JtuWD9ZK9xtr0pBX5XssZbJ4gKkv5mY0GXp/2+5pKiCs9T6h5ZXIOdhqY28QGavwBDWstEALlIk5RxrkfSTUV654l9SlKKnGAInLuaQuuqSx4GLG6mVExZLdi9TQVYEBdswKExO9ZR1YAkY+W5+Arc2zIakqAROzsN83BH08d9Irnszo8q0P7kiJEAJVMu721bzgpbTCtWniOah59VIOgD6xyhKJKsecZi4GnGuGYiofPQvwEhBhVTOnWkcn9SDH+a0EtPvj69vdbh/T+/vzu5+evbdZK308FvH6WHAqAPISh6+Rnw0L/FguroSPSLEzEv/yM9ofZaCyQKf5rh3Mt23aH+WSR2l0i65F/hFBZB4bqEUtVM1I7bAlpRtTagKiOXFFLJ5VVYrsZNhu6qgNxugJWL1BGA/ml9JfcKxJNhr80pMWw0TSVkxqr7Qcxm4X3gXqxpg7UzW8gG1ge8aGdn3WPDhE27hhQZB1WAgO/I5Y0lBqy8OwN/iG38nxk4SymLWLNjLTc/vOr2PzTKux7UHNjO0iOCywkBANjFzo+QZWtnFKdfeRYLs7P+l4RkZk7HhSmz5GzZL5nmKnHENiebWFYnHFHooM25zS9gsMX2TByx5fcuJOWswgtIeg9VbnYEOOv1gIDQkpm3rE1sfAtoHyfGIx8B0mCOarfLtA4xv7fYrXzrltRK1u49gfOGKbvHvXX8+FoAVoHQNUtmuswPqvlT+vwzFXrrEXufwL7eY2w6ktYDE793/HNN9q/3TIUSLKNwY8jGNhTODbEArpRb+HrN8T/h1+UpwfuGrtKy3pzNoRBOlrV8HbWt1NZ80qAcCCuAwmEwuH6lQ13q5/l2HY5HbR9IALBNaMbxu16I1+LoO5VlKS8HNY96BjuMOpse2oEpnLR0yOPewrZzEw3edSnFckfQwqELlcYetJSYesLH4tqTRrcdl2sOZCXra3VQGjeUXD8Yo1orn/bKrPl618IQy//Bw==
\ No newline at end of file
diff --git a/core-writer/external/CMakeLists.txt b/core-writer/external/CMakeLists.txt
deleted file mode 100644
index 993a66a..0000000
--- a/core-writer/external/CMakeLists.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-file(GLOB SOURCES
- bitshuffle/*.c
- bitshuffle/*.cpp)
-
-add_library(external STATIC ${SOURCES})
-target_include_directories(external PUBLIC .)
-include_directories(/usr/include)
\ No newline at end of file
diff --git a/core-writer/external/bitshuffle/bitshuffle.cpp b/core-writer/external/bitshuffle/bitshuffle.cpp
deleted file mode 100644
index b8944d7..0000000
--- a/core-writer/external/bitshuffle/bitshuffle.cpp
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Bitshuffle - Filter for improving compression of typed binary data.
- *
- * Author: Kiyoshi Masui
- * Website: http://www.github.com/kiyo-masui/bitshuffle
- * Created: 2014
- *
- * See LICENSE file for details about copyright and rights to use.
- *
- */
-
-#include "bitshuffle.h"
-#include "bitshuffle_core.h"
-
-extern "C" {
- #include "bitshuffle_internals.h"
- #include "lz4.h"
-}
-
-
-#include
-#include
-
-
-// Constants.
-// Use fast decompression instead of safe decompression for LZ4.
-#define BSHUF_LZ4_DECOMPRESS_FAST
-
-
-// Macros.
-#define CHECK_ERR_FREE_LZ(count, buf) if (count < 0) { \
- free(buf); return count - 1000; }
-
-
-/* Bitshuffle and compress a single block. */
-int64_t bshuf_compress_lz4_block(ioc_chain *C_ptr, \
- const size_t size, const size_t elem_size) {
-
- int64_t nbytes, count;
- void *tmp_buf_bshuf;
- void *tmp_buf_lz4;
- size_t this_iter;
- const void *in;
- void *out;
-
- tmp_buf_bshuf = malloc(size * elem_size);
- if (tmp_buf_bshuf == NULL) return -1;
-
- tmp_buf_lz4 = malloc(LZ4_compressBound(size * elem_size));
- if (tmp_buf_lz4 == NULL){
- free(tmp_buf_bshuf);
- return -1;
- }
-
-
- in = ioc_get_in(C_ptr, &this_iter);
- ioc_set_next_in(C_ptr, &this_iter, (void*) ((char*) in + size * elem_size));
-
- count = bshuf_trans_bit_elem(in, tmp_buf_bshuf, size, elem_size);
- if (count < 0) {
- free(tmp_buf_lz4);
- free(tmp_buf_bshuf);
- return count;
- }
- nbytes = LZ4_compress((const char*) tmp_buf_bshuf, (char*) tmp_buf_lz4, size * elem_size);
- free(tmp_buf_bshuf);
- CHECK_ERR_FREE_LZ(nbytes, tmp_buf_lz4);
-
- out = ioc_get_out(C_ptr, &this_iter);
- ioc_set_next_out(C_ptr, &this_iter, (void *) ((char *) out + nbytes + 4));
-
- bshuf_write_uint32_BE(out, nbytes);
- memcpy((char *) out + 4, tmp_buf_lz4, nbytes);
-
- free(tmp_buf_lz4);
-
- return nbytes + 4;
-}
-
-
-/* Decompress and bitunshuffle a single block. */
-int64_t bshuf_decompress_lz4_block(ioc_chain *C_ptr,
- const size_t size, const size_t elem_size) {
-
- int64_t nbytes, count;
- void *out, *tmp_buf;
- const void *in;
- size_t this_iter;
- int32_t nbytes_from_header;
-
- in = ioc_get_in(C_ptr, &this_iter);
- nbytes_from_header = bshuf_read_uint32_BE(in);
- ioc_set_next_in(C_ptr, &this_iter,
- (void*) ((char*) in + nbytes_from_header + 4));
-
- out = ioc_get_out(C_ptr, &this_iter);
- ioc_set_next_out(C_ptr, &this_iter,
- (void *) ((char *) out + size * elem_size));
-
- tmp_buf = malloc(size * elem_size);
- if (tmp_buf == NULL) return -1;
-
-#ifdef BSHUF_LZ4_DECOMPRESS_FAST
- nbytes = LZ4_decompress_fast((const char*) in + 4, (char*) tmp_buf, size * elem_size);
- CHECK_ERR_FREE_LZ(nbytes, tmp_buf);
- if (nbytes != nbytes_from_header) {
- free(tmp_buf);
- return -91;
- }
-#else
- nbytes = LZ4_decompress_safe((const char*) in + 4, (char *) tmp_buf, nbytes_from_header,
- size * elem_size);
- CHECK_ERR_FREE_LZ(nbytes, tmp_buf);
- if (nbytes != size * elem_size) {
- free(tmp_buf);
- return -91;
- }
- nbytes = nbytes_from_header;
-#endif
- count = bshuf_untrans_bit_elem(tmp_buf, out, size, elem_size);
- CHECK_ERR_FREE(count, tmp_buf);
- nbytes += 4;
-
- free(tmp_buf);
- return nbytes;
-}
-
-
-/* ---- Public functions ----
- *
- * See header file for description and usage.
- *
- */
-
-size_t bshuf_compress_lz4_bound(const size_t size,
- const size_t elem_size, size_t block_size) {
-
- size_t bound, leftover;
-
- if (block_size == 0) {
- block_size = bshuf_default_block_size(elem_size);
- }
- if (block_size % BSHUF_BLOCKED_MULT) return -81;
-
- // Note that each block gets a 4 byte header.
- // Size of full blocks.
- bound = (LZ4_compressBound(block_size * elem_size) + 4) * (size / block_size);
- // Size of partial blocks, if any.
- leftover = ((size % block_size) / BSHUF_BLOCKED_MULT) * BSHUF_BLOCKED_MULT;
- if (leftover) bound += LZ4_compressBound(leftover * elem_size) + 4;
- // Size of uncompressed data not fitting into any blocks.
- bound += (size % BSHUF_BLOCKED_MULT) * elem_size;
- return bound;
-}
-
-
-int64_t bshuf_compress_lz4(const void* in, void* out, const size_t size,
- const size_t elem_size, size_t block_size) {
- return bshuf_blocked_wrap_fun(&bshuf_compress_lz4_block, in, out, size,
- elem_size, block_size);
-}
-
-
-int64_t bshuf_decompress_lz4(const void* in, void* out, const size_t size,
- const size_t elem_size, size_t block_size) {
- return bshuf_blocked_wrap_fun(&bshuf_decompress_lz4_block, in, out, size,
- elem_size, block_size);
-}
-
diff --git a/core-writer/external/bitshuffle/bitshuffle.h b/core-writer/external/bitshuffle/bitshuffle.h
deleted file mode 100644
index 3df95f4..0000000
--- a/core-writer/external/bitshuffle/bitshuffle.h
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Bitshuffle - Filter for improving compression of typed binary data.
- *
- * This file is part of Bitshuffle
- * Author: Kiyoshi Masui
- * Website: http://www.github.com/kiyo-masui/bitshuffle
- * Created: 2014
- *
- * See LICENSE file for details about copyright and rights to use.
- *
- *
- * Header File
- *
- * Worker routines return an int64_t which is the number of bytes processed
- * if positive or an error code if negative.
- *
- * Error codes:
- * -1 : Failed to allocate memory.
- * -11 : Missing SSE.
- * -12 : Missing AVX.
- * -80 : Input size not a multiple of 8.
- * -81 : block_size not multiple of 8.
- * -91 : Decompression error, wrong number of bytes processed.
- * -1YYY : Error internal to compression routine with error code -YYY.
- */
-
-
-#ifndef BITSHUFFLE_H
-#define BITSHUFFLE_H
-
-#include
-#include "bitshuffle_core.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* ---- bshuf_compress_lz4_bound ----
- *
- * Bound on size of data compressed with *bshuf_compress_lz4*.
- *
- * Parameters
- * ----------
- * size : number of elements in input
- * elem_size : element size of typed data
- * block_size : Process in blocks of this many elements. Pass 0 to
- * select automatically (recommended).
- *
- * Returns
- * -------
- * Bound on compressed data size.
- *
- */
-size_t bshuf_compress_lz4_bound(const size_t size,
- const size_t elem_size, size_t block_size);
-
-
-/* ---- bshuf_compress_lz4 ----
- *
- * Bitshuffled and compress the data using LZ4.
- *
- * Transpose within elements, in blocks of data of *block_size* elements then
- * compress the blocks using LZ4. In the output buffer, each block is prefixed
- * by a 4 byte integer giving the compressed size of that block.
- *
- * Output buffer must be large enough to hold the compressed data. This could
- * be in principle substantially larger than the input buffer. Use the routine
- * *bshuf_compress_lz4_bound* to get an upper limit.
- *
- * Parameters
- * ----------
- * in : input buffer, must be of size * elem_size bytes
- * out : output buffer, must be large enough to hold data.
- * size : number of elements in input
- * elem_size : element size of typed data
- * block_size : Process in blocks of this many elements. Pass 0 to
- * select automatically (recommended).
- *
- * Returns
- * -------
- * number of bytes used in output buffer, negative error-code if failed.
- *
- */
-int64_t bshuf_compress_lz4(const void* in, void* out, const size_t size, const size_t
- elem_size, size_t block_size);
-
-
-/* ---- bshuf_decompress_lz4 ----
- *
- * Undo compression and bitshuffling.
- *
- * Decompress data then un-bitshuffle it in blocks of *block_size* elements.
- *
- * To properly unshuffle bitshuffled data, *size*, *elem_size* and *block_size*
- * must patch the parameters used to compress the data.
- *
- * NOT TO BE USED WITH UNTRUSTED DATA: This routine uses the function
- * LZ4_decompress_fast from LZ4, which does not protect against maliciously
- * formed datasets. By modifying the compressed data, this function could be
- * coerced into leaving the boundaries of the input buffer.
- *
- * Parameters
- * ----------
- * in : input buffer
- * out : output buffer, must be of size * elem_size bytes
- * size : number of elements in input
- * elem_size : element size of typed data
- * block_size : Process in blocks of this many elements. Pass 0 to
- * select automatically (recommended).
- *
- * Returns
- * -------
- * number of bytes consumed in *input* buffer, negative error-code if failed.
- *
- */
-int64_t bshuf_decompress_lz4(const void* in, void* out, const size_t size,
- const size_t elem_size, size_t block_size);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif
-
-#endif // BITSHUFFLE_H
diff --git a/core-writer/external/bitshuffle/bitshuffle_core.cpp b/core-writer/external/bitshuffle/bitshuffle_core.cpp
deleted file mode 100644
index 539197f..0000000
--- a/core-writer/external/bitshuffle/bitshuffle_core.cpp
+++ /dev/null
@@ -1,1336 +0,0 @@
-/*
- * Bitshuffle - Filter for improving compression of typed binary data.
- *
- * Author: Kiyoshi Masui
- * Website: http://www.github.com/kiyo-masui/bitshuffle
- * Created: 2014
- *
- * See LICENSE file for details about copyright and rights to use.
- *
- */
-
-#include "bitshuffle_core.h"
-
-extern "C" {
- #include "bitshuffle_internals.h"
-}
-
-#include
-#include
-
-
-#if defined(__AVX2__) && defined (__SSE2__)
-#define USEAVX2
-#endif
-
-#if defined(__SSE2__)
-#define USESSE2
-#endif
-
-
-// Conditional includes for SSE2 and AVX2.
-#ifdef USEAVX2
-#include
-#elif defined USESSE2
-#include
-#endif
-
-
-// Macros.
-#define CHECK_MULT_EIGHT(n) if (n % 8) return -80;
-#define MAX(X,Y) ((X) > (Y) ? (X) : (Y))
-
-
-/* ---- Functions indicating compile time instruction set. ---- */
-
-int bshuf_using_SSE2(void) {
-#ifdef USESSE2
- return 1;
-#else
- return 0;
-#endif
-}
-
-
-int bshuf_using_AVX2(void) {
-#ifdef USEAVX2
- return 1;
-#else
- return 0;
-#endif
-}
-
-
-/* ---- Worker code not requiring special instruction sets. ----
- *
- * The following code does not use any x86 specific vectorized instructions
- * and should compile on any machine
- *
- */
-
-/* Transpose 8x8 bit array packed into a single quadword *x*.
- * *t* is workspace. */
-#define TRANS_BIT_8X8(x, t) { \
- t = (x ^ (x >> 7)) & 0x00AA00AA00AA00AALL; \
- x = x ^ t ^ (t << 7); \
- t = (x ^ (x >> 14)) & 0x0000CCCC0000CCCCLL; \
- x = x ^ t ^ (t << 14); \
- t = (x ^ (x >> 28)) & 0x00000000F0F0F0F0LL; \
- x = x ^ t ^ (t << 28); \
- }
-
-/* Transpose 8x8 bit array along the diagonal from upper right
- to lower left */
-#define TRANS_BIT_8X8_BE(x, t) { \
- t = (x ^ (x >> 9)) & 0x0055005500550055LL; \
- x = x ^ t ^ (t << 9); \
- t = (x ^ (x >> 18)) & 0x0000333300003333LL; \
- x = x ^ t ^ (t << 18); \
- t = (x ^ (x >> 36)) & 0x000000000F0F0F0FLL; \
- x = x ^ t ^ (t << 36); \
- }
-
-/* Transpose of an array of arbitrarily typed elements. */
-#define TRANS_ELEM_TYPE(in, out, lda, ldb, type_t) { \
- size_t ii, jj, kk; \
- const type_t* in_type = (const type_t*) in; \
- type_t* out_type = (type_t*) out; \
- for(ii = 0; ii + 7 < lda; ii += 8) { \
- for(jj = 0; jj < ldb; jj++) { \
- for(kk = 0; kk < 8; kk++) { \
- out_type[jj*lda + ii + kk] = \
- in_type[ii*ldb + kk * ldb + jj]; \
- } \
- } \
- } \
- for(ii = lda - lda % 8; ii < lda; ii ++) { \
- for(jj = 0; jj < ldb; jj++) { \
- out_type[jj*lda + ii] = in_type[ii*ldb + jj]; \
- } \
- } \
- }
-
-
-/* Memory copy with bshuf call signature. For testing and profiling. */
-int64_t bshuf_copy(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- const char* in_b = (const char*) in;
- char* out_b = (char*) out;
-
- memcpy(out_b, in_b, size * elem_size);
- return size * elem_size;
-}
-
-
-/* Transpose bytes within elements, starting partway through input. */
-int64_t bshuf_trans_byte_elem_remainder(const void* in, void* out, const size_t size,
- const size_t elem_size, const size_t start) {
-
- size_t ii, jj, kk;
- const char* in_b = (const char*) in;
- char* out_b = (char*) out;
-
- CHECK_MULT_EIGHT(start);
-
- if (size > start) {
- // ii loop separated into 2 loops so the compiler can unroll
- // the inner one.
- for (ii = start; ii + 7 < size; ii += 8) {
- for (jj = 0; jj < elem_size; jj++) {
- for (kk = 0; kk < 8; kk++) {
- out_b[jj * size + ii + kk]
- = in_b[ii * elem_size + kk * elem_size + jj];
- }
- }
- }
- for (ii = size - size % 8; ii < size; ii ++) {
- for (jj = 0; jj < elem_size; jj++) {
- out_b[jj * size + ii] = in_b[ii * elem_size + jj];
- }
- }
- }
- return size * elem_size;
-}
-
-
-/* Transpose bytes within elements. */
-int64_t bshuf_trans_byte_elem_scal(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- return bshuf_trans_byte_elem_remainder(in, out, size, elem_size, 0);
-}
-
-
-/* Transpose bits within bytes. */
-int64_t bshuf_trans_bit_byte_remainder(const void* in, void* out, const size_t size,
- const size_t elem_size, const size_t start_byte) {
-
- const uint64_t* in_b = (const uint64_t*) in;
- uint8_t* out_b = (uint8_t*) out;
-
- uint64_t x, t;
-
- size_t ii, kk;
- size_t nbyte = elem_size * size;
- size_t nbyte_bitrow = nbyte / 8;
-
- uint64_t e=1;
- const int little_endian = *(uint8_t *) &e == 1;
- const size_t bit_row_skip = little_endian ? nbyte_bitrow : -nbyte_bitrow;
- const int64_t bit_row_offset = little_endian ? 0 : 7 * nbyte_bitrow;
-
- CHECK_MULT_EIGHT(nbyte);
- CHECK_MULT_EIGHT(start_byte);
-
- for (ii = start_byte / 8; ii < nbyte_bitrow; ii ++) {
- x = in_b[ii];
- if (little_endian) {
- TRANS_BIT_8X8(x, t);
- } else {
- TRANS_BIT_8X8_BE(x, t);
- }
- for (kk = 0; kk < 8; kk ++) {
- out_b[bit_row_offset + kk * bit_row_skip + ii] = x;
- x = x >> 8;
- }
- }
- return size * elem_size;
-}
-
-
-/* Transpose bits within bytes. */
-int64_t bshuf_trans_bit_byte_scal(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- return bshuf_trans_bit_byte_remainder(in, out, size, elem_size, 0);
-}
-
-
-/* General transpose of an array, optimized for large element sizes. */
-int64_t bshuf_trans_elem(const void* in, void* out, const size_t lda,
- const size_t ldb, const size_t elem_size) {
-
- size_t ii, jj;
- const char* in_b = (const char*) in;
- char* out_b = (char*) out;
- for(ii = 0; ii < lda; ii++) {
- for(jj = 0; jj < ldb; jj++) {
- memcpy(&out_b[(jj*lda + ii) * elem_size],
- &in_b[(ii*ldb + jj) * elem_size], elem_size);
- }
- }
- return lda * ldb * elem_size;
-}
-
-
-/* Transpose rows of shuffled bits (size / 8 bytes) within groups of 8. */
-int64_t bshuf_trans_bitrow_eight(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- size_t nbyte_bitrow = size / 8;
-
- CHECK_MULT_EIGHT(size);
-
- return bshuf_trans_elem(in, out, 8, elem_size, nbyte_bitrow);
-}
-
-
-/* Transpose bits within elements. */
-int64_t bshuf_trans_bit_elem_scal(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- int64_t count;
- void *tmp_buf;
-
- CHECK_MULT_EIGHT(size);
-
- tmp_buf = malloc(size * elem_size);
- if (tmp_buf == NULL) return -1;
-
- count = bshuf_trans_byte_elem_scal(in, out, size, elem_size);
- CHECK_ERR_FREE(count, tmp_buf);
- count = bshuf_trans_bit_byte_scal(out, tmp_buf, size, elem_size);
- CHECK_ERR_FREE(count, tmp_buf);
- count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size);
-
- free(tmp_buf);
-
- return count;
-}
-
-
-/* For data organized into a row for each bit (8 * elem_size rows), transpose
- * the bytes. */
-int64_t bshuf_trans_byte_bitrow_scal(const void* in, void* out, const size_t size,
- const size_t elem_size) {
- size_t ii, jj, kk, nbyte_row;
- const char *in_b;
- char *out_b;
-
-
- in_b = (const char*) in;
- out_b = (char*) out;
-
- nbyte_row = size / 8;
-
- CHECK_MULT_EIGHT(size);
-
- for (jj = 0; jj < elem_size; jj++) {
- for (ii = 0; ii < nbyte_row; ii++) {
- for (kk = 0; kk < 8; kk++) {
- out_b[ii * 8 * elem_size + jj * 8 + kk] = \
- in_b[(jj * 8 + kk) * nbyte_row + ii];
- }
- }
- }
- return size * elem_size;
-}
-
-
-/* Shuffle bits within the bytes of eight element blocks. */
-int64_t bshuf_shuffle_bit_eightelem_scal(const void* in, void* out, \
- const size_t size, const size_t elem_size) {
-
- const char *in_b;
- char *out_b;
- uint64_t x, t;
- size_t ii, jj, kk;
- size_t nbyte, out_index;
-
- uint64_t e=1;
- const int little_endian = *(uint8_t *) &e == 1;
- const size_t elem_skip = little_endian ? elem_size : -elem_size;
- const uint64_t elem_offset = little_endian ? 0 : 7 * elem_size;
-
- CHECK_MULT_EIGHT(size);
-
- in_b = (const char*) in;
- out_b = (char*) out;
-
- nbyte = elem_size * size;
-
- for (jj = 0; jj < 8 * elem_size; jj += 8) {
- for (ii = 0; ii + 8 * elem_size - 1 < nbyte; ii += 8 * elem_size) {
- x = *((uint64_t*) &in_b[ii + jj]);
- if (little_endian) {
- TRANS_BIT_8X8(x, t);
- } else {
- TRANS_BIT_8X8_BE(x, t);
- }
- for (kk = 0; kk < 8; kk++) {
- out_index = ii + jj / 8 + elem_offset + kk * elem_skip;
- *((uint8_t*) &out_b[out_index]) = x;
- x = x >> 8;
- }
- }
- }
- return size * elem_size;
-}
-
-
-/* Untranspose bits within elements. */
-int64_t bshuf_untrans_bit_elem_scal(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- int64_t count;
- void *tmp_buf;
-
- CHECK_MULT_EIGHT(size);
-
- tmp_buf = malloc(size * elem_size);
- if (tmp_buf == NULL) return -1;
-
- count = bshuf_trans_byte_bitrow_scal(in, tmp_buf, size, elem_size);
- CHECK_ERR_FREE(count, tmp_buf);
- count = bshuf_shuffle_bit_eightelem_scal(tmp_buf, out, size, elem_size);
-
- free(tmp_buf);
-
- return count;
-}
-
-
-/* ---- Worker code that uses SSE2 ----
- *
- * The following code makes use of the SSE2 instruction set and specialized
- * 16 byte registers. The SSE2 instructions are present on modern x86
- * processors. The first Intel processor microarchitecture supporting SSE2 was
- * Pentium 4 (2000).
- *
- */
-
-#ifdef USESSE2
-
-/* Transpose bytes within elements for 16 bit elements. */
-int64_t bshuf_trans_byte_elem_SSE_16(const void* in, void* out, const size_t size) {
-
- size_t ii;
- const char *in_b = (const char*) in;
- char *out_b = (char*) out;
- __m128i a0, b0, a1, b1;
-
- for (ii=0; ii + 15 < size; ii += 16) {
- a0 = _mm_loadu_si128((__m128i *) &in_b[2*ii + 0*16]);
- b0 = _mm_loadu_si128((__m128i *) &in_b[2*ii + 1*16]);
-
- a1 = _mm_unpacklo_epi8(a0, b0);
- b1 = _mm_unpackhi_epi8(a0, b0);
-
- a0 = _mm_unpacklo_epi8(a1, b1);
- b0 = _mm_unpackhi_epi8(a1, b1);
-
- a1 = _mm_unpacklo_epi8(a0, b0);
- b1 = _mm_unpackhi_epi8(a0, b0);
-
- a0 = _mm_unpacklo_epi8(a1, b1);
- b0 = _mm_unpackhi_epi8(a1, b1);
-
- _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0);
- _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0);
- }
- return bshuf_trans_byte_elem_remainder(in, out, size, 2,
- size - size % 16);
-}
-
-
-/* Transpose bytes within elements for 32 bit elements. */
-int64_t bshuf_trans_byte_elem_SSE_32(const void* in, void* out, const size_t size) {
-
- size_t ii;
- const char *in_b;
- char *out_b;
- in_b = (const char*) in;
- out_b = (char*) out;
- __m128i a0, b0, c0, d0, a1, b1, c1, d1;
-
- for (ii=0; ii + 15 < size; ii += 16) {
- a0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 0*16]);
- b0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 1*16]);
- c0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 2*16]);
- d0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 3*16]);
-
- a1 = _mm_unpacklo_epi8(a0, b0);
- b1 = _mm_unpackhi_epi8(a0, b0);
- c1 = _mm_unpacklo_epi8(c0, d0);
- d1 = _mm_unpackhi_epi8(c0, d0);
-
- a0 = _mm_unpacklo_epi8(a1, b1);
- b0 = _mm_unpackhi_epi8(a1, b1);
- c0 = _mm_unpacklo_epi8(c1, d1);
- d0 = _mm_unpackhi_epi8(c1, d1);
-
- a1 = _mm_unpacklo_epi8(a0, b0);
- b1 = _mm_unpackhi_epi8(a0, b0);
- c1 = _mm_unpacklo_epi8(c0, d0);
- d1 = _mm_unpackhi_epi8(c0, d0);
-
- a0 = _mm_unpacklo_epi64(a1, c1);
- b0 = _mm_unpackhi_epi64(a1, c1);
- c0 = _mm_unpacklo_epi64(b1, d1);
- d0 = _mm_unpackhi_epi64(b1, d1);
-
- _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0);
- _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0);
- _mm_storeu_si128((__m128i *) &out_b[2*size + ii], c0);
- _mm_storeu_si128((__m128i *) &out_b[3*size + ii], d0);
- }
- return bshuf_trans_byte_elem_remainder(in, out, size, 4,
- size - size % 16);
-}
-
-
-/* Transpose bytes within elements for 64 bit elements. */
-int64_t bshuf_trans_byte_elem_SSE_64(const void* in, void* out, const size_t size) {
-
- size_t ii;
- const char* in_b = (const char*) in;
- char* out_b = (char*) out;
- __m128i a0, b0, c0, d0, e0, f0, g0, h0;
- __m128i a1, b1, c1, d1, e1, f1, g1, h1;
-
- for (ii=0; ii + 15 < size; ii += 16) {
- a0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 0*16]);
- b0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 1*16]);
- c0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 2*16]);
- d0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 3*16]);
- e0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 4*16]);
- f0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 5*16]);
- g0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 6*16]);
- h0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 7*16]);
-
- a1 = _mm_unpacklo_epi8(a0, b0);
- b1 = _mm_unpackhi_epi8(a0, b0);
- c1 = _mm_unpacklo_epi8(c0, d0);
- d1 = _mm_unpackhi_epi8(c0, d0);
- e1 = _mm_unpacklo_epi8(e0, f0);
- f1 = _mm_unpackhi_epi8(e0, f0);
- g1 = _mm_unpacklo_epi8(g0, h0);
- h1 = _mm_unpackhi_epi8(g0, h0);
-
- a0 = _mm_unpacklo_epi8(a1, b1);
- b0 = _mm_unpackhi_epi8(a1, b1);
- c0 = _mm_unpacklo_epi8(c1, d1);
- d0 = _mm_unpackhi_epi8(c1, d1);
- e0 = _mm_unpacklo_epi8(e1, f1);
- f0 = _mm_unpackhi_epi8(e1, f1);
- g0 = _mm_unpacklo_epi8(g1, h1);
- h0 = _mm_unpackhi_epi8(g1, h1);
-
- a1 = _mm_unpacklo_epi32(a0, c0);
- b1 = _mm_unpackhi_epi32(a0, c0);
- c1 = _mm_unpacklo_epi32(b0, d0);
- d1 = _mm_unpackhi_epi32(b0, d0);
- e1 = _mm_unpacklo_epi32(e0, g0);
- f1 = _mm_unpackhi_epi32(e0, g0);
- g1 = _mm_unpacklo_epi32(f0, h0);
- h1 = _mm_unpackhi_epi32(f0, h0);
-
- a0 = _mm_unpacklo_epi64(a1, e1);
- b0 = _mm_unpackhi_epi64(a1, e1);
- c0 = _mm_unpacklo_epi64(b1, f1);
- d0 = _mm_unpackhi_epi64(b1, f1);
- e0 = _mm_unpacklo_epi64(c1, g1);
- f0 = _mm_unpackhi_epi64(c1, g1);
- g0 = _mm_unpacklo_epi64(d1, h1);
- h0 = _mm_unpackhi_epi64(d1, h1);
-
- _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0);
- _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0);
- _mm_storeu_si128((__m128i *) &out_b[2*size + ii], c0);
- _mm_storeu_si128((__m128i *) &out_b[3*size + ii], d0);
- _mm_storeu_si128((__m128i *) &out_b[4*size + ii], e0);
- _mm_storeu_si128((__m128i *) &out_b[5*size + ii], f0);
- _mm_storeu_si128((__m128i *) &out_b[6*size + ii], g0);
- _mm_storeu_si128((__m128i *) &out_b[7*size + ii], h0);
- }
- return bshuf_trans_byte_elem_remainder(in, out, size, 8,
- size - size % 16);
-}
-
-
-/* Transpose bytes within elements using best SSE algorithm available. */
-int64_t bshuf_trans_byte_elem_SSE(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- int64_t count;
-
- // Trivial cases: power of 2 bytes.
- switch (elem_size) {
- case 1:
- count = bshuf_copy(in, out, size, elem_size);
- return count;
- case 2:
- count = bshuf_trans_byte_elem_SSE_16(in, out, size);
- return count;
- case 4:
- count = bshuf_trans_byte_elem_SSE_32(in, out, size);
- return count;
- case 8:
- count = bshuf_trans_byte_elem_SSE_64(in, out, size);
- return count;
- }
-
- // Worst case: odd number of bytes. Turns out that this is faster for
- // (odd * 2) byte elements as well (hence % 4).
- if (elem_size % 4) {
- count = bshuf_trans_byte_elem_scal(in, out, size, elem_size);
- return count;
- }
-
- // Multiple of power of 2: transpose hierarchically.
- {
- size_t nchunk_elem;
- void* tmp_buf = malloc(size * elem_size);
- if (tmp_buf == NULL) return -1;
-
- if ((elem_size % 8) == 0) {
- nchunk_elem = elem_size / 8;
- TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int64_t);
- count = bshuf_trans_byte_elem_SSE_64(out, tmp_buf,
- size * nchunk_elem);
- bshuf_trans_elem(tmp_buf, out, 8, nchunk_elem, size);
- } else if ((elem_size % 4) == 0) {
- nchunk_elem = elem_size / 4;
- TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int32_t);
- count = bshuf_trans_byte_elem_SSE_32(out, tmp_buf,
- size * nchunk_elem);
- bshuf_trans_elem(tmp_buf, out, 4, nchunk_elem, size);
- } else {
- // Not used since scalar algorithm is faster.
- nchunk_elem = elem_size / 2;
- TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int16_t);
- count = bshuf_trans_byte_elem_SSE_16(out, tmp_buf,
- size * nchunk_elem);
- bshuf_trans_elem(tmp_buf, out, 2, nchunk_elem, size);
- }
-
- free(tmp_buf);
- return count;
- }
-}
-
-
-/* Transpose bits within bytes. */
-int64_t bshuf_trans_bit_byte_SSE(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- size_t ii, kk;
- const char* in_b = (const char*) in;
- char* out_b = (char*) out;
- uint16_t* out_ui16;
-
- int64_t count;
-
- size_t nbyte = elem_size * size;
-
- CHECK_MULT_EIGHT(nbyte);
-
- __m128i xmm;
- int32_t bt;
-
- for (ii = 0; ii + 15 < nbyte; ii += 16) {
- xmm = _mm_loadu_si128((__m128i *) &in_b[ii]);
- for (kk = 0; kk < 8; kk++) {
- bt = _mm_movemask_epi8(xmm);
- xmm = _mm_slli_epi16(xmm, 1);
- out_ui16 = (uint16_t*) &out_b[((7 - kk) * nbyte + ii) / 8];
- *out_ui16 = bt;
- }
- }
- count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size,
- nbyte - nbyte % 16);
- return count;
-}
-
-
-/* Transpose bits within elements. */
-int64_t bshuf_trans_bit_elem_SSE(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- int64_t count;
-
- CHECK_MULT_EIGHT(size);
-
- void* tmp_buf = malloc(size * elem_size);
- if (tmp_buf == NULL) return -1;
-
- count = bshuf_trans_byte_elem_SSE(in, out, size, elem_size);
- CHECK_ERR_FREE(count, tmp_buf);
- count = bshuf_trans_bit_byte_SSE(out, tmp_buf, size, elem_size);
- CHECK_ERR_FREE(count, tmp_buf);
- count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size);
-
- free(tmp_buf);
-
- return count;
-}
-
-
-/* For data organized into a row for each bit (8 * elem_size rows), transpose
- * the bytes. */
-int64_t bshuf_trans_byte_bitrow_SSE(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- size_t ii, jj;
- const char* in_b = (const char*) in;
- char* out_b = (char*) out;
-
- CHECK_MULT_EIGHT(size);
-
- size_t nrows = 8 * elem_size;
- size_t nbyte_row = size / 8;
-
- __m128i a0, b0, c0, d0, e0, f0, g0, h0;
- __m128i a1, b1, c1, d1, e1, f1, g1, h1;
- __m128 *as, *bs, *cs, *ds, *es, *fs, *gs, *hs;
-
- for (ii = 0; ii + 7 < nrows; ii += 8) {
- for (jj = 0; jj + 15 < nbyte_row; jj += 16) {
- a0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 0)*nbyte_row + jj]);
- b0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 1)*nbyte_row + jj]);
- c0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 2)*nbyte_row + jj]);
- d0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 3)*nbyte_row + jj]);
- e0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 4)*nbyte_row + jj]);
- f0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 5)*nbyte_row + jj]);
- g0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 6)*nbyte_row + jj]);
- h0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 7)*nbyte_row + jj]);
-
-
- a1 = _mm_unpacklo_epi8(a0, b0);
- b1 = _mm_unpacklo_epi8(c0, d0);
- c1 = _mm_unpacklo_epi8(e0, f0);
- d1 = _mm_unpacklo_epi8(g0, h0);
- e1 = _mm_unpackhi_epi8(a0, b0);
- f1 = _mm_unpackhi_epi8(c0, d0);
- g1 = _mm_unpackhi_epi8(e0, f0);
- h1 = _mm_unpackhi_epi8(g0, h0);
-
-
- a0 = _mm_unpacklo_epi16(a1, b1);
- b0 = _mm_unpacklo_epi16(c1, d1);
- c0 = _mm_unpackhi_epi16(a1, b1);
- d0 = _mm_unpackhi_epi16(c1, d1);
-
- e0 = _mm_unpacklo_epi16(e1, f1);
- f0 = _mm_unpacklo_epi16(g1, h1);
- g0 = _mm_unpackhi_epi16(e1, f1);
- h0 = _mm_unpackhi_epi16(g1, h1);
-
-
- a1 = _mm_unpacklo_epi32(a0, b0);
- b1 = _mm_unpackhi_epi32(a0, b0);
-
- c1 = _mm_unpacklo_epi32(c0, d0);
- d1 = _mm_unpackhi_epi32(c0, d0);
-
- e1 = _mm_unpacklo_epi32(e0, f0);
- f1 = _mm_unpackhi_epi32(e0, f0);
-
- g1 = _mm_unpacklo_epi32(g0, h0);
- h1 = _mm_unpackhi_epi32(g0, h0);
-
- // We don't have a storeh instruction for integers, so interpret
- // as a float. Have a storel (_mm_storel_epi64).
- as = (__m128 *) &a1;
- bs = (__m128 *) &b1;
- cs = (__m128 *) &c1;
- ds = (__m128 *) &d1;
- es = (__m128 *) &e1;
- fs = (__m128 *) &f1;
- gs = (__m128 *) &g1;
- hs = (__m128 *) &h1;
-
- _mm_storel_pi((__m64 *) &out_b[(jj + 0) * nrows + ii], *as);
- _mm_storel_pi((__m64 *) &out_b[(jj + 2) * nrows + ii], *bs);
- _mm_storel_pi((__m64 *) &out_b[(jj + 4) * nrows + ii], *cs);
- _mm_storel_pi((__m64 *) &out_b[(jj + 6) * nrows + ii], *ds);
- _mm_storel_pi((__m64 *) &out_b[(jj + 8) * nrows + ii], *es);
- _mm_storel_pi((__m64 *) &out_b[(jj + 10) * nrows + ii], *fs);
- _mm_storel_pi((__m64 *) &out_b[(jj + 12) * nrows + ii], *gs);
- _mm_storel_pi((__m64 *) &out_b[(jj + 14) * nrows + ii], *hs);
-
- _mm_storeh_pi((__m64 *) &out_b[(jj + 1) * nrows + ii], *as);
- _mm_storeh_pi((__m64 *) &out_b[(jj + 3) * nrows + ii], *bs);
- _mm_storeh_pi((__m64 *) &out_b[(jj + 5) * nrows + ii], *cs);
- _mm_storeh_pi((__m64 *) &out_b[(jj + 7) * nrows + ii], *ds);
- _mm_storeh_pi((__m64 *) &out_b[(jj + 9) * nrows + ii], *es);
- _mm_storeh_pi((__m64 *) &out_b[(jj + 11) * nrows + ii], *fs);
- _mm_storeh_pi((__m64 *) &out_b[(jj + 13) * nrows + ii], *gs);
- _mm_storeh_pi((__m64 *) &out_b[(jj + 15) * nrows + ii], *hs);
- }
- for (jj = nbyte_row - nbyte_row % 16; jj < nbyte_row; jj ++) {
- out_b[jj * nrows + ii + 0] = in_b[(ii + 0)*nbyte_row + jj];
- out_b[jj * nrows + ii + 1] = in_b[(ii + 1)*nbyte_row + jj];
- out_b[jj * nrows + ii + 2] = in_b[(ii + 2)*nbyte_row + jj];
- out_b[jj * nrows + ii + 3] = in_b[(ii + 3)*nbyte_row + jj];
- out_b[jj * nrows + ii + 4] = in_b[(ii + 4)*nbyte_row + jj];
- out_b[jj * nrows + ii + 5] = in_b[(ii + 5)*nbyte_row + jj];
- out_b[jj * nrows + ii + 6] = in_b[(ii + 6)*nbyte_row + jj];
- out_b[jj * nrows + ii + 7] = in_b[(ii + 7)*nbyte_row + jj];
- }
- }
- return size * elem_size;
-}
-
-
-/* Shuffle bits within the bytes of eight element blocks. */
-int64_t bshuf_shuffle_bit_eightelem_SSE(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- CHECK_MULT_EIGHT(size);
-
- // With a bit of care, this could be written such that such that it is
- // in_buf = out_buf safe.
- const char* in_b = (const char*) in;
- uint16_t* out_ui16 = (uint16_t*) out;
-
- size_t ii, jj, kk;
- size_t nbyte = elem_size * size;
-
- __m128i xmm;
- int32_t bt;
-
- if (elem_size % 2) {
- bshuf_shuffle_bit_eightelem_scal(in, out, size, elem_size);
- } else {
- for (ii = 0; ii + 8 * elem_size - 1 < nbyte;
- ii += 8 * elem_size) {
- for (jj = 0; jj + 15 < 8 * elem_size; jj += 16) {
- xmm = _mm_loadu_si128((__m128i *) &in_b[ii + jj]);
- for (kk = 0; kk < 8; kk++) {
- bt = _mm_movemask_epi8(xmm);
- xmm = _mm_slli_epi16(xmm, 1);
- size_t ind = (ii + jj / 8 + (7 - kk) * elem_size);
- out_ui16[ind / 2] = bt;
- }
- }
- }
- }
- return size * elem_size;
-}
-
-
-/* Untranspose bits within elements. */
-int64_t bshuf_untrans_bit_elem_SSE(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- int64_t count;
-
- CHECK_MULT_EIGHT(size);
-
- void* tmp_buf = malloc(size * elem_size);
- if (tmp_buf == NULL) return -1;
-
- count = bshuf_trans_byte_bitrow_SSE(in, tmp_buf, size, elem_size);
- CHECK_ERR_FREE(count, tmp_buf);
- count = bshuf_shuffle_bit_eightelem_SSE(tmp_buf, out, size, elem_size);
-
- free(tmp_buf);
-
- return count;
-}
-
-#else // #ifdef USESSE2
-
-
-int64_t bshuf_untrans_bit_elem_SSE(const void* in, void* out, const size_t size,
- const size_t elem_size) {
- return -11;
-}
-
-
-int64_t bshuf_trans_bit_elem_SSE(const void* in, void* out, const size_t size,
- const size_t elem_size) {
- return -11;
-}
-
-
-int64_t bshuf_trans_byte_bitrow_SSE(const void* in, void* out, const size_t size,
- const size_t elem_size) {
- return -11;
-}
-
-
-int64_t bshuf_trans_bit_byte_SSE(const void* in, void* out, const size_t size,
- const size_t elem_size) {
- return -11;
-}
-
-
-int64_t bshuf_trans_byte_elem_SSE(const void* in, void* out, const size_t size,
- const size_t elem_size) {
- return -11;
-}
-
-
-int64_t bshuf_trans_byte_elem_SSE_64(const void* in, void* out, const size_t size) {
- return -11;
-}
-
-
-int64_t bshuf_trans_byte_elem_SSE_32(const void* in, void* out, const size_t size) {
- return -11;
-}
-
-
-int64_t bshuf_trans_byte_elem_SSE_16(const void* in, void* out, const size_t size) {
- return -11;
-}
-
-
-int64_t bshuf_shuffle_bit_eightelem_SSE(const void* in, void* out, const size_t size,
- const size_t elem_size) {
- return -11;
-}
-
-
-#endif // #ifdef USESSE2
-
-
-/* ---- Code that requires AVX2. Intel Haswell (2013) and later. ---- */
-
-/* ---- Worker code that uses AVX2 ----
- *
- * The following code makes use of the AVX2 instruction set and specialized
- * 32 byte registers. The AVX2 instructions are present on newer x86
- * processors. The first Intel processor microarchitecture supporting AVX2 was
- * Haswell (2013).
- *
- */
-
-#ifdef USEAVX2
-
-/* Transpose bits within bytes. */
-int64_t bshuf_trans_bit_byte_AVX(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- size_t ii, kk;
- const char* in_b = (const char*) in;
- char* out_b = (char*) out;
- int32_t* out_i32;
-
- size_t nbyte = elem_size * size;
-
- int64_t count;
-
- __m256i ymm;
- int32_t bt;
-
- for (ii = 0; ii + 31 < nbyte; ii += 32) {
- ymm = _mm256_loadu_si256((__m256i *) &in_b[ii]);
- for (kk = 0; kk < 8; kk++) {
- bt = _mm256_movemask_epi8(ymm);
- ymm = _mm256_slli_epi16(ymm, 1);
- out_i32 = (int32_t*) &out_b[((7 - kk) * nbyte + ii) / 8];
- *out_i32 = bt;
- }
- }
- count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size,
- nbyte - nbyte % 32);
- return count;
-}
-
-
-/* Transpose bits within elements. */
-int64_t bshuf_trans_bit_elem_AVX(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- int64_t count;
-
- CHECK_MULT_EIGHT(size);
-
- void* tmp_buf = malloc(size * elem_size);
- if (tmp_buf == NULL) return -1;
-
- count = bshuf_trans_byte_elem_SSE(in, out, size, elem_size);
- CHECK_ERR_FREE(count, tmp_buf);
- count = bshuf_trans_bit_byte_AVX(out, tmp_buf, size, elem_size);
- CHECK_ERR_FREE(count, tmp_buf);
- count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size);
-
- free(tmp_buf);
-
- return count;
-}
-
-
-/* For data organized into a row for each bit (8 * elem_size rows), transpose
- * the bytes. */
-int64_t bshuf_trans_byte_bitrow_AVX(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- size_t hh, ii, jj, kk, mm;
- const char* in_b = (const char*) in;
- char* out_b = (char*) out;
-
- CHECK_MULT_EIGHT(size);
-
- size_t nrows = 8 * elem_size;
- size_t nbyte_row = size / 8;
-
- if (elem_size % 4) return bshuf_trans_byte_bitrow_SSE(in, out, size,
- elem_size);
-
- __m256i ymm_0[8];
- __m256i ymm_1[8];
- __m256i ymm_storeage[8][4];
-
- for (jj = 0; jj + 31 < nbyte_row; jj += 32) {
- for (ii = 0; ii + 3 < elem_size; ii += 4) {
- for (hh = 0; hh < 4; hh ++) {
-
- for (kk = 0; kk < 8; kk ++){
- ymm_0[kk] = _mm256_loadu_si256((__m256i *) &in_b[
- (ii * 8 + hh * 8 + kk) * nbyte_row + jj]);
- }
-
- for (kk = 0; kk < 4; kk ++){
- ymm_1[kk] = _mm256_unpacklo_epi8(ymm_0[kk * 2],
- ymm_0[kk * 2 + 1]);
- ymm_1[kk + 4] = _mm256_unpackhi_epi8(ymm_0[kk * 2],
- ymm_0[kk * 2 + 1]);
- }
-
- for (kk = 0; kk < 2; kk ++){
- for (mm = 0; mm < 2; mm ++){
- ymm_0[kk * 4 + mm] = _mm256_unpacklo_epi16(
- ymm_1[kk * 4 + mm * 2],
- ymm_1[kk * 4 + mm * 2 + 1]);
- ymm_0[kk * 4 + mm + 2] = _mm256_unpackhi_epi16(
- ymm_1[kk * 4 + mm * 2],
- ymm_1[kk * 4 + mm * 2 + 1]);
- }
- }
-
- for (kk = 0; kk < 4; kk ++){
- ymm_1[kk * 2] = _mm256_unpacklo_epi32(ymm_0[kk * 2],
- ymm_0[kk * 2 + 1]);
- ymm_1[kk * 2 + 1] = _mm256_unpackhi_epi32(ymm_0[kk * 2],
- ymm_0[kk * 2 + 1]);
- }
-
- for (kk = 0; kk < 8; kk ++){
- ymm_storeage[kk][hh] = ymm_1[kk];
- }
- }
-
- for (mm = 0; mm < 8; mm ++) {
-
- for (kk = 0; kk < 4; kk ++){
- ymm_0[kk] = ymm_storeage[mm][kk];
- }
-
- ymm_1[0] = _mm256_unpacklo_epi64(ymm_0[0], ymm_0[1]);
- ymm_1[1] = _mm256_unpacklo_epi64(ymm_0[2], ymm_0[3]);
- ymm_1[2] = _mm256_unpackhi_epi64(ymm_0[0], ymm_0[1]);
- ymm_1[3] = _mm256_unpackhi_epi64(ymm_0[2], ymm_0[3]);
-
- ymm_0[0] = _mm256_permute2x128_si256(ymm_1[0], ymm_1[1], 32);
- ymm_0[1] = _mm256_permute2x128_si256(ymm_1[2], ymm_1[3], 32);
- ymm_0[2] = _mm256_permute2x128_si256(ymm_1[0], ymm_1[1], 49);
- ymm_0[3] = _mm256_permute2x128_si256(ymm_1[2], ymm_1[3], 49);
-
- _mm256_storeu_si256((__m256i *) &out_b[
- (jj + mm * 2 + 0 * 16) * nrows + ii * 8], ymm_0[0]);
- _mm256_storeu_si256((__m256i *) &out_b[
- (jj + mm * 2 + 0 * 16 + 1) * nrows + ii * 8], ymm_0[1]);
- _mm256_storeu_si256((__m256i *) &out_b[
- (jj + mm * 2 + 1 * 16) * nrows + ii * 8], ymm_0[2]);
- _mm256_storeu_si256((__m256i *) &out_b[
- (jj + mm * 2 + 1 * 16 + 1) * nrows + ii * 8], ymm_0[3]);
- }
- }
- }
- for (ii = 0; ii < nrows; ii ++ ) {
- for (jj = nbyte_row - nbyte_row % 32; jj < nbyte_row; jj ++) {
- out_b[jj * nrows + ii] = in_b[ii * nbyte_row + jj];
- }
- }
- return size * elem_size;
-}
-
-
-/* Shuffle bits within the bytes of eight element blocks. */
-int64_t bshuf_shuffle_bit_eightelem_AVX(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- CHECK_MULT_EIGHT(size);
-
- // With a bit of care, this could be written such that such that it is
- // in_buf = out_buf safe.
- const char* in_b = (const char*) in;
- char* out_b = (char*) out;
-
- size_t ii, jj, kk;
- size_t nbyte = elem_size * size;
-
- __m256i ymm;
- int32_t bt;
-
- if (elem_size % 4) {
- return bshuf_shuffle_bit_eightelem_SSE(in, out, size, elem_size);
- } else {
- for (jj = 0; jj + 31 < 8 * elem_size; jj += 32) {
- for (ii = 0; ii + 8 * elem_size - 1 < nbyte;
- ii += 8 * elem_size) {
- ymm = _mm256_loadu_si256((__m256i *) &in_b[ii + jj]);
- for (kk = 0; kk < 8; kk++) {
- bt = _mm256_movemask_epi8(ymm);
- ymm = _mm256_slli_epi16(ymm, 1);
- size_t ind = (ii + jj / 8 + (7 - kk) * elem_size);
- * (int32_t *) &out_b[ind] = bt;
- }
- }
- }
- }
- return size * elem_size;
-}
-
-
-/* Untranspose bits within elements. */
-int64_t bshuf_untrans_bit_elem_AVX(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- int64_t count;
-
- CHECK_MULT_EIGHT(size);
-
- void* tmp_buf = malloc(size * elem_size);
- if (tmp_buf == NULL) return -1;
-
- count = bshuf_trans_byte_bitrow_AVX(in, tmp_buf, size, elem_size);
- CHECK_ERR_FREE(count, tmp_buf);
- count = bshuf_shuffle_bit_eightelem_AVX(tmp_buf, out, size, elem_size);
-
- free(tmp_buf);
- return count;
-}
-
-
-#else // #ifdef USEAVX2
-
-int64_t bshuf_trans_bit_byte_AVX(const void* in, void* out, const size_t size,
- const size_t elem_size) {
- return -12;
-}
-
-
-int64_t bshuf_trans_bit_elem_AVX(const void* in, void* out, const size_t size,
- const size_t elem_size) {
- return -12;
-}
-
-
-int64_t bshuf_trans_byte_bitrow_AVX(const void* in, void* out, const size_t size,
- const size_t elem_size) {
- return -12;
-}
-
-
-int64_t bshuf_shuffle_bit_eightelem_AVX(const void* in, void* out, const size_t size,
- const size_t elem_size) {
- return -12;
-}
-
-
-int64_t bshuf_untrans_bit_elem_AVX(const void* in, void* out, const size_t size,
- const size_t elem_size) {
- return -12;
-}
-
-#endif // #ifdef USEAVX2
-
-
-/* ---- Drivers selecting best instruction set at compile time. ---- */
-
-int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- int64_t count;
-#ifdef USEAVX2
- count = bshuf_trans_bit_elem_AVX(in, out, size, elem_size);
-#elif defined(USESSE2)
- count = bshuf_trans_bit_elem_SSE(in, out, size, elem_size);
-#else
- count = bshuf_trans_bit_elem_scal(in, out, size, elem_size);
-#endif
- return count;
-}
-
-
-int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size,
- const size_t elem_size) {
-
- int64_t count;
-#ifdef USEAVX2
- count = bshuf_untrans_bit_elem_AVX(in, out, size, elem_size);
-#elif defined(USESSE2)
- count = bshuf_untrans_bit_elem_SSE(in, out, size, elem_size);
-#else
- count = bshuf_untrans_bit_elem_scal(in, out, size, elem_size);
-#endif
- return count;
-}
-
-
-/* ---- Wrappers for implementing blocking ---- */
-
-/* Wrap a function for processing a single block to process an entire buffer in
- * parallel. */
-int64_t bshuf_blocked_wrap_fun(bshufBlockFunDef fun, const void* in, void* out, \
- const size_t size, const size_t elem_size, size_t block_size) {
-
- size_t ii;
- int64_t err = 0;
- int64_t count, cum_count=0;
- size_t last_block_size;
- size_t leftover_bytes;
- size_t this_iter;
- char *last_in;
- char *last_out;
-
-
- ioc_chain C;
- ioc_init(&C, in, out);
-
-
- if (block_size == 0) {
- block_size = bshuf_default_block_size(elem_size);
- }
- if (block_size % BSHUF_BLOCKED_MULT) return -81;
-
-#if defined(_OPENMP)
- #pragma omp parallel for schedule(dynamic, 1) \
- private(count) reduction(+ : cum_count)
-#endif
- for (ii = 0; ii < size / block_size; ii ++) {
- count = fun(&C, block_size, elem_size);
- if (count < 0) err = count;
- cum_count += count;
- }
-
- last_block_size = size % block_size;
- last_block_size = last_block_size - last_block_size % BSHUF_BLOCKED_MULT;
- if (last_block_size) {
- count = fun(&C, last_block_size, elem_size);
- if (count < 0) err = count;
- cum_count += count;
- }
-
- if (err < 0) return err;
-
- leftover_bytes = size % BSHUF_BLOCKED_MULT * elem_size;
- //this_iter;
- last_in = (char *) ioc_get_in(&C, &this_iter);
- ioc_set_next_in(&C, &this_iter, (void *) (last_in + leftover_bytes));
- last_out = (char *) ioc_get_out(&C, &this_iter);
- ioc_set_next_out(&C, &this_iter, (void *) (last_out + leftover_bytes));
-
- memcpy(last_out, last_in, leftover_bytes);
-
- ioc_destroy(&C);
-
- return cum_count + leftover_bytes;
-}
-
-
-/* Bitshuffle a single block. */
-int64_t bshuf_bitshuffle_block(ioc_chain *C_ptr, \
- const size_t size, const size_t elem_size) {
-
- size_t this_iter;
- const void *in;
- void *out;
- int64_t count;
-
-
-
- in = ioc_get_in(C_ptr, &this_iter);
- ioc_set_next_in(C_ptr, &this_iter,
- (void*) ((char*) in + size * elem_size));
- out = ioc_get_out(C_ptr, &this_iter);
- ioc_set_next_out(C_ptr, &this_iter,
- (void *) ((char *) out + size * elem_size));
-
- count = bshuf_trans_bit_elem(in, out, size, elem_size);
- return count;
-}
-
-
-/* Bitunshuffle a single block. */
-int64_t bshuf_bitunshuffle_block(ioc_chain* C_ptr, \
- const size_t size, const size_t elem_size) {
-
-
- size_t this_iter;
- const void *in;
- void *out;
- int64_t count;
-
-
-
-
- in = ioc_get_in(C_ptr, &this_iter);
- ioc_set_next_in(C_ptr, &this_iter,
- (void*) ((char*) in + size * elem_size));
- out = ioc_get_out(C_ptr, &this_iter);
- ioc_set_next_out(C_ptr, &this_iter,
- (void *) ((char *) out + size * elem_size));
-
- count = bshuf_untrans_bit_elem(in, out, size, elem_size);
- return count;
-}
-
-
-/* Write a 64 bit unsigned integer to a buffer in big endian order. */
-void bshuf_write_uint64_BE(void* buf, uint64_t num) {
- int ii;
- uint8_t* b = (uint8_t*) buf;
- uint64_t pow28 = 1 << 8;
- for (ii = 7; ii >= 0; ii--) {
- b[ii] = num % pow28;
- num = num / pow28;
- }
-}
-
-
-/* Read a 64 bit unsigned integer from a buffer big endian order. */
-uint64_t bshuf_read_uint64_BE(void* buf) {
- int ii;
- uint8_t* b = (uint8_t*) buf;
- uint64_t num = 0, pow28 = 1 << 8, cp = 1;
- for (ii = 7; ii >= 0; ii--) {
- num += b[ii] * cp;
- cp *= pow28;
- }
- return num;
-}
-
-
-/* Write a 32 bit unsigned integer to a buffer in big endian order. */
-void bshuf_write_uint32_BE(void* buf, uint32_t num) {
- int ii;
- uint8_t* b = (uint8_t*) buf;
- uint32_t pow28 = 1 << 8;
- for (ii = 3; ii >= 0; ii--) {
- b[ii] = num % pow28;
- num = num / pow28;
- }
-}
-
-
-/* Read a 32 bit unsigned integer from a buffer big endian order. */
-uint32_t bshuf_read_uint32_BE(const void* buf) {
- int ii;
- uint8_t* b = (uint8_t*) buf;
- uint32_t num = 0, pow28 = 1 << 8, cp = 1;
- for (ii = 3; ii >= 0; ii--) {
- num += b[ii] * cp;
- cp *= pow28;
- }
- return num;
-}
-
-
-/* ---- Public functions ----
- *
- * See header file for description and usage.
- *
- */
-
-size_t bshuf_default_block_size(const size_t elem_size) {
- // This function needs to be absolutely stable between versions.
- // Otherwise encoded data will not be decodable.
-
- size_t block_size = BSHUF_TARGET_BLOCK_SIZE_B / elem_size;
- // Ensure it is a required multiple.
- block_size = (block_size / BSHUF_BLOCKED_MULT) * BSHUF_BLOCKED_MULT;
- return MAX(block_size, BSHUF_MIN_RECOMMEND_BLOCK);
-}
-
-
-int64_t bshuf_bitshuffle(const void* in, void* out, const size_t size,
- const size_t elem_size, size_t block_size) {
-
- return bshuf_blocked_wrap_fun(&bshuf_bitshuffle_block, in, out, size,
- elem_size, block_size);
-}
-
-
-int64_t bshuf_bitunshuffle(const void* in, void* out, const size_t size,
- const size_t elem_size, size_t block_size) {
-
- return bshuf_blocked_wrap_fun(&bshuf_bitunshuffle_block, in, out, size,
- elem_size, block_size);
-}
-
-
-#undef TRANS_BIT_8X8
-#undef TRANS_ELEM_TYPE
-#undef MAX
-#undef CHECK_MULT_EIGHT
-#undef CHECK_ERR_FREE
-
-#undef USESSE2
-#undef USEAVX2
diff --git a/core-writer/external/bitshuffle/bitshuffle_core.h b/core-writer/external/bitshuffle/bitshuffle_core.h
deleted file mode 100644
index 0250c04..0000000
--- a/core-writer/external/bitshuffle/bitshuffle_core.h
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Bitshuffle - Filter for improving compression of typed binary data.
- *
- * This file is part of Bitshuffle
- * Author: Kiyoshi Masui
- * Website: http://www.github.com/kiyo-masui/bitshuffle
- * Created: 2014
- *
- * See LICENSE file for details about copyright and rights to use.
- *
- *
- * Header File
- *
- * Worker routines return an int64_t which is the number of bytes processed
- * if positive or an error code if negative.
- *
- * Error codes:
- * -1 : Failed to allocate memory.
- * -11 : Missing SSE.
- * -12 : Missing AVX.
- * -80 : Input size not a multiple of 8.
- * -81 : block_size not multiple of 8.
- * -91 : Decompression error, wrong number of bytes processed.
- * -1YYY : Error internal to compression routine with error code -YYY.
- */
-
-
-#ifndef BITSHUFFLE_CORE_H
-#define BITSHUFFLE_CORE_H
-
-// We assume GNU g++ defining `__cplusplus` has stdint.h
-//#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199900L) || defined(__cplusplus)
-#include
-//#else
-// typedef unsigned char uint8_t;
-// typedef unsigned short uint16_t;
-// typedef unsigned int uint32_t;
-// typedef signed int int32_t;
-// typedef unsigned long long uint64_t;
-// typedef long long int64_t;
-//#endif
-
-#include
-
-
-// These are usually set in the setup.py.
-#ifndef BSHUF_VERSION_MAJOR
-#define BSHUF_VERSION_MAJOR 0
-#define BSHUF_VERSION_MINOR 3
-#define BSHUF_VERSION_POINT 3
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* --- bshuf_using_SSE2 ----
- *
- * Whether routines where compiled with the SSE2 instruction set.
- *
- * Returns
- * -------
- * 1 if using SSE2, 0 otherwise.
- *
- */
-int bshuf_using_SSE2(void);
-
-
-/* ---- bshuf_using_AVX2 ----
- *
- * Whether routines where compiled with the AVX2 instruction set.
- *
- * Returns
- * -------
- * 1 if using AVX2, 0 otherwise.
- *
- */
-int bshuf_using_AVX2(void);
-
-
-/* ---- bshuf_default_block_size ----
- *
- * The default block size as function of element size.
- *
- * This is the block size used by the blocked routines (any routine
- * taking a *block_size* argument) when the block_size is not provided
- * (zero is passed).
- *
- * The results of this routine are guaranteed to be stable such that
- * shuffled/compressed data can always be decompressed.
- *
- * Parameters
- * ----------
- * elem_size : element size of data to be shuffled/compressed.
- *
- */
-size_t bshuf_default_block_size(const size_t elem_size);
-
-
-/* ---- bshuf_bitshuffle ----
- *
- * Bitshuffle the data.
- *
- * Transpose the bits within elements, in blocks of *block_size*
- * elements.
- *
- * Parameters
- * ----------
- * in : input buffer, must be of size * elem_size bytes
- * out : output buffer, must be of size * elem_size bytes
- * size : number of elements in input
- * elem_size : element size of typed data
- * block_size : Do transpose in blocks of this many elements. Pass 0 to
- * select automatically (recommended).
- *
- * Returns
- * -------
- * number of bytes processed, negative error-code if failed.
- *
- */
-int64_t bshuf_bitshuffle(const void* in, void* out, const size_t size,
- const size_t elem_size, size_t block_size);
-
-
-/* ---- bshuf_bitunshuffle ----
- *
- * Unshuffle bitshuffled data.
- *
- * Untranspose the bits within elements, in blocks of *block_size*
- * elements.
- *
- * To properly unshuffle bitshuffled data, *size*, *elem_size* and *block_size*
- * must match the parameters used to shuffle the data.
- *
- * Parameters
- * ----------
- * in : input buffer, must be of size * elem_size bytes
- * out : output buffer, must be of size * elem_size bytes
- * size : number of elements in input
- * elem_size : element size of typed data
- * block_size : Do transpose in blocks of this many elements. Pass 0 to
- * select automatically (recommended).
- *
- * Returns
- * -------
- * number of bytes processed, negative error-code if failed.
- *
- */
-int64_t bshuf_bitunshuffle(const void* in, void* out, const size_t size,
- const size_t elem_size, size_t block_size);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif
-
-#endif // BITSHUFFLE_CORE_H
diff --git a/core-writer/external/bitshuffle/bitshuffle_internals.h b/core-writer/external/bitshuffle/bitshuffle_internals.h
deleted file mode 100644
index 7e9810e..0000000
--- a/core-writer/external/bitshuffle/bitshuffle_internals.h
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Bitshuffle - Filter for improving compression of typed binary data.
- *
- * This file is part of Bitshuffle
- * Author: Kiyoshi Masui
- * Website: http://www.github.com/kiyo-masui/bitshuffle
- * Created: 2014
- *
- * See LICENSE file for details about copyright and rights to use.
- */
-
-
-#ifndef BITSHUFFLE_INTERNALS_H
-#define BITSHUFFLE_INTERNALS_H
-
-// We assume GNU g++ defining `__cplusplus` has stdint.h
-//#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199900L) || defined(__cplusplus)
-#include
-//#else
-// typedef unsigned char uint8_t;
-// typedef unsigned short uint16_t;
-// typedef unsigned int uint32_t;
-// typedef signed int int32_t;
-// typedef unsigned long long uint64_t;
-// typedef long long int64_t;
-//#endif
-
-#include
-#include "iochain.h"
-
-
-// Constants.
-#ifndef BSHUF_MIN_RECOMMEND_BLOCK
-#define BSHUF_MIN_RECOMMEND_BLOCK 128
-#define BSHUF_BLOCKED_MULT 8 // Block sizes must be multiple of this.
-#define BSHUF_TARGET_BLOCK_SIZE_B 8192
-#endif
-
-
-// Macros.
-#define CHECK_ERR_FREE(count, buf) if (count < 0) { free(buf); return count; }
-
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* ---- Utility functions for internal use only ---- */
-
-int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size,
- const size_t elem_size);
-
-/* Read a 32 bit unsigned integer from a buffer big endian order. */
-uint32_t bshuf_read_uint32_BE(const void* buf);
-
-/* Write a 32 bit unsigned integer to a buffer in big endian order. */
-void bshuf_write_uint32_BE(void* buf, uint32_t num);
-
-int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size,
- const size_t elem_size);
-
-/* Function definition for worker functions that process a single block. */
-typedef int64_t (*bshufBlockFunDef)(ioc_chain* C_ptr,
- const size_t size, const size_t elem_size);
-
-/* Wrap a function for processing a single block to process an entire buffer in
- * parallel. */
-int64_t bshuf_blocked_wrap_fun(bshufBlockFunDef fun, const void* in, void* out,
- const size_t size, const size_t elem_size, size_t block_size);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif
-
-#endif // BITSHUFFLE_INTERNALS_H
diff --git a/core-writer/external/bitshuffle/iochain.c b/core-writer/external/bitshuffle/iochain.c
deleted file mode 100644
index baa9729..0000000
--- a/core-writer/external/bitshuffle/iochain.c
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * IOchain - Distribute a chain of dependant IO events amoung threads.
- *
- * This file is part of Bitshuffle
- * Author: Kiyoshi Masui
- * Website: http://www.github.com/kiyo-masui/bitshuffle
- * Created: 2014
- *
- * See LICENSE file for details about copyright and rights to use.
- *
- */
-
-#include
-#include "iochain.h"
-
-
-void ioc_init(ioc_chain *C, const void *in_ptr_0, void *out_ptr_0) {
-#ifdef _OPENMP
- omp_init_lock(&C->next_lock);
- for (size_t ii = 0; ii < IOC_SIZE; ii ++) {
- omp_init_lock(&(C->in_pl[ii].lock));
- omp_init_lock(&(C->out_pl[ii].lock));
- }
-#endif
- C->next = 0;
- C->in_pl[0].ptr = in_ptr_0;
- C->out_pl[0].ptr = out_ptr_0;
-}
-
-
-void ioc_destroy(ioc_chain *C) {
-#ifdef _OPENMP
- omp_destroy_lock(&C->next_lock);
- for (size_t ii = 0; ii < IOC_SIZE; ii ++) {
- omp_destroy_lock(&(C->in_pl[ii].lock));
- omp_destroy_lock(&(C->out_pl[ii].lock));
- }
-#endif
-}
-
-
-const void * ioc_get_in(ioc_chain *C, size_t *this_iter) {
-#ifdef _OPENMP
- omp_set_lock(&C->next_lock);
- #pragma omp flush
-#endif
- *this_iter = C->next;
- C->next ++;
-#ifdef _OPENMP
- omp_set_lock(&(C->in_pl[*this_iter % IOC_SIZE].lock));
- omp_set_lock(&(C->in_pl[(*this_iter + 1) % IOC_SIZE].lock));
- omp_set_lock(&(C->out_pl[(*this_iter + 1) % IOC_SIZE].lock));
- omp_unset_lock(&C->next_lock);
-#endif
- return C->in_pl[*this_iter % IOC_SIZE].ptr;
-}
-
-
-void ioc_set_next_in(ioc_chain *C, size_t* this_iter, void* in_ptr) {
- C->in_pl[(*this_iter + 1) % IOC_SIZE].ptr = in_ptr;
-#ifdef _OPENMP
- omp_unset_lock(&(C->in_pl[(*this_iter + 1) % IOC_SIZE].lock));
-#endif
-}
-
-
-void * ioc_get_out(ioc_chain *C, size_t *this_iter) {
-#ifdef _OPENMP
- omp_set_lock(&(C->out_pl[(*this_iter) % IOC_SIZE].lock));
- #pragma omp flush
-#endif
- void *out_ptr = C->out_pl[*this_iter % IOC_SIZE].ptr;
-#ifdef _OPENMP
- omp_unset_lock(&(C->out_pl[(*this_iter) % IOC_SIZE].lock));
-#endif
- return out_ptr;
-}
-
-
-void ioc_set_next_out(ioc_chain *C, size_t *this_iter, void* out_ptr) {
- C->out_pl[(*this_iter + 1) % IOC_SIZE].ptr = out_ptr;
-#ifdef _OPENMP
- omp_unset_lock(&(C->out_pl[(*this_iter + 1) % IOC_SIZE].lock));
- // *in_pl[this_iter]* lock released at the end of the iteration to avoid being
- // overtaken by previous threads and having *out_pl[this_iter]* corrupted.
- // Especially worried about thread 0, iteration 0.
- omp_unset_lock(&(C->in_pl[(*this_iter) % IOC_SIZE].lock));
-#endif
-}
-
diff --git a/core-writer/external/bitshuffle/iochain.h b/core-writer/external/bitshuffle/iochain.h
deleted file mode 100644
index 4e225d1..0000000
--- a/core-writer/external/bitshuffle/iochain.h
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * IOchain - Distribute a chain of dependant IO events amoung threads.
- *
- * This file is part of Bitshuffle
- * Author: Kiyoshi Masui
- * Website: http://www.github.com/kiyo-masui/bitshuffle
- * Created: 2014
- *
- * See LICENSE file for details about copyright and rights to use.
- *
- *
- * Header File
- *
- * Similar in concept to a queue. Each task includes reading an input
- * and writing output, but the location of the input/output (the pointers)
- * depend on the previous item in the chain.
- *
- * This is designed for parallelizing blocked compression/decompression IO,
- * where the destination of a compressed block depends on the compressed size
- * of all previous blocks.
- *
- * Implemented with OpenMP locks.
- *
- *
- * Usage
- * -----
- * - Call `ioc_init` in serial block.
- * - Each thread should create a local variable *size_t this_iter* and
- * pass its address to all function calls. Its value will be set
- * inside the functions and is used to identify the thread.
- * - Each thread must call each of the `ioc_get*` and `ioc_set*` methods
- * exactly once per iteration, starting with `ioc_get_in` and ending
- * with `ioc_set_next_out`.
- * - The order (`ioc_get_in`, `ioc_set_next_in`, *work*, `ioc_get_out`,
- * `ioc_set_next_out`, *work*) is most efficient.
- * - Have each thread call `ioc_end_pop`.
- * - `ioc_get_in` is blocked until the previous entry's
- * `ioc_set_next_in` is called.
- * - `ioc_get_out` is blocked until the previous entry's
- * `ioc_set_next_out` is called.
- * - There are no blocks on the very first iteration.
- * - Call `ioc_destroy` in serial block.
- * - Safe for num_threads >= IOC_SIZE (but less efficient).
- *
- */
-
-
-#ifndef IOCHAIN_H
-#define IOCHAIN_H
-
-
-#include
-#ifdef _OPENMP
-#include
-#endif
-
-
-#define IOC_SIZE 33
-
-
-typedef struct ioc_ptr_and_lock {
-#ifdef _OPENMP
- omp_lock_t lock;
-#endif
- void *ptr;
-} ptr_and_lock;
-
-typedef struct ioc_const_ptr_and_lock {
-#ifdef _OPENMP
- omp_lock_t lock;
-#endif
- const void *ptr;
-} const_ptr_and_lock;
-
-
-typedef struct ioc_chain {
-#ifdef _OPENMP
- omp_lock_t next_lock;
-#endif
- size_t next;
- const_ptr_and_lock in_pl[IOC_SIZE];
- ptr_and_lock out_pl[IOC_SIZE];
-} ioc_chain;
-
-
-void ioc_init(ioc_chain *C, const void *in_ptr_0, void *out_ptr_0);
-void ioc_destroy(ioc_chain *C);
-const void * ioc_get_in(ioc_chain *C, size_t *this_iter);
-void ioc_set_next_in(ioc_chain *C, size_t* this_iter, void* in_ptr);
-void * ioc_get_out(ioc_chain *C, size_t *this_iter);
-void ioc_set_next_out(ioc_chain *C, size_t *this_iter, void* out_ptr);
-
-#endif // IOCHAIN_H
-
diff --git a/core-writer/external/bitshuffle/lz4.c b/core-writer/external/bitshuffle/lz4.c
deleted file mode 100644
index 08cf6b5..0000000
--- a/core-writer/external/bitshuffle/lz4.c
+++ /dev/null
@@ -1,1516 +0,0 @@
-/*
- LZ4 - Fast LZ compression algorithm
- Copyright (C) 2011-2015, Yann Collet.
-
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - LZ4 source repository : https://github.com/Cyan4973/lz4
- - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
-
-
-/**************************************
-* Tuning parameters
-**************************************/
-/*
- * HEAPMODE :
- * Select how default compression functions will allocate memory for their hash table,
- * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
- */
-#define HEAPMODE 0
-
-/*
- * ACCELERATION_DEFAULT :
- * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0
- */
-#define ACCELERATION_DEFAULT 1
-
-
-/**************************************
-* CPU Feature Detection
-**************************************/
-/*
- * LZ4_FORCE_SW_BITCOUNT
- * Define this parameter if your target system or compiler does not support hardware bit count
- */
-#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */
-# define LZ4_FORCE_SW_BITCOUNT
-#endif
-
-
-/**************************************
-* Includes
-**************************************/
-#include "lz4.h"
-
-
-/**************************************
-* Compiler Options
-**************************************/
-#ifdef _MSC_VER /* Visual Studio */
-# define FORCE_INLINE static __forceinline
-# include
-# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */
-#else
-# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
-# if defined(__GNUC__) || defined(__clang__)
-# define FORCE_INLINE static inline __attribute__((always_inline))
-# else
-# define FORCE_INLINE static inline
-# endif
-# else
-# define FORCE_INLINE static
-# endif /* __STDC_VERSION__ */
-#endif /* _MSC_VER */
-
-/* LZ4_GCC_VERSION is defined into lz4.h */
-#if (LZ4_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__)
-# define expect(expr,value) (__builtin_expect ((expr),(value)) )
-#else
-# define expect(expr,value) (expr)
-#endif
-
-#define likely(expr) expect((expr) != 0, 1)
-#define unlikely(expr) expect((expr) != 0, 0)
-
-
-/**************************************
-* Memory routines
-**************************************/
-#include /* malloc, calloc, free */
-#define ALLOCATOR(n,s) calloc(n,s)
-#define FREEMEM free
-#include /* memset, memcpy */
-#define MEM_INIT memset
-
-
-/**************************************
-* Basic Types
-**************************************/
-#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
-# include
- typedef uint8_t BYTE;
- typedef uint16_t U16;
- typedef uint32_t U32;
- typedef int32_t S32;
- typedef uint64_t U64;
-#else
- typedef unsigned char BYTE;
- typedef unsigned short U16;
- typedef unsigned int U32;
- typedef signed int S32;
- typedef unsigned long long U64;
-#endif
-
-
-/**************************************
-* Reading and writing into memory
-**************************************/
-#define STEPSIZE sizeof(size_t)
-
-static unsigned LZ4_64bits(void) { return sizeof(void*)==8; }
-
-static unsigned LZ4_isLittleEndian(void)
-{
- const union { U32 i; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
- return one.c[0];
-}
-
-
-static U16 LZ4_read16(const void* memPtr)
-{
- U16 val16;
- memcpy(&val16, memPtr, 2);
- return val16;
-}
-
-static U16 LZ4_readLE16(const void* memPtr)
-{
- if (LZ4_isLittleEndian())
- {
- return LZ4_read16(memPtr);
- }
- else
- {
- const BYTE* p = (const BYTE*)memPtr;
- return (U16)((U16)p[0] + (p[1]<<8));
- }
-}
-
-static void LZ4_writeLE16(void* memPtr, U16 value)
-{
- if (LZ4_isLittleEndian())
- {
- memcpy(memPtr, &value, 2);
- }
- else
- {
- BYTE* p = (BYTE*)memPtr;
- p[0] = (BYTE) value;
- p[1] = (BYTE)(value>>8);
- }
-}
-
-static U32 LZ4_read32(const void* memPtr)
-{
- U32 val32;
- memcpy(&val32, memPtr, 4);
- return val32;
-}
-
-static U64 LZ4_read64(const void* memPtr)
-{
- U64 val64;
- memcpy(&val64, memPtr, 8);
- return val64;
-}
-
-static size_t LZ4_read_ARCH(const void* p)
-{
- if (LZ4_64bits())
- return (size_t)LZ4_read64(p);
- else
- return (size_t)LZ4_read32(p);
-}
-
-
-static void LZ4_copy4(void* dstPtr, const void* srcPtr) { memcpy(dstPtr, srcPtr, 4); }
-
-static void LZ4_copy8(void* dstPtr, const void* srcPtr) { memcpy(dstPtr, srcPtr, 8); }
-
-/* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */
-static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
-{
- BYTE* d = (BYTE*)dstPtr;
- const BYTE* s = (const BYTE*)srcPtr;
- BYTE* e = (BYTE*)dstEnd;
- do { LZ4_copy8(d,s); d+=8; s+=8; } while (d>3);
-# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_ctzll((U64)val) >> 3);
-# else
- static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
- return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
-# endif
- }
- else /* 32 bits */
- {
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
- unsigned long r;
- _BitScanForward( &r, (U32)val );
- return (int)(r>>3);
-# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_ctz((U32)val) >> 3);
-# else
- static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
- return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
-# endif
- }
- }
- else /* Big Endian CPU */
- {
- if (LZ4_64bits())
- {
-# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT)
- unsigned long r = 0;
- _BitScanReverse64( &r, val );
- return (unsigned)(r>>3);
-# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_clzll((U64)val) >> 3);
-# else
- unsigned r;
- if (!(val>>32)) { r=4; } else { r=0; val>>=32; }
- if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
- r += (!val);
- return r;
-# endif
- }
- else /* 32 bits */
- {
-# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT)
- unsigned long r = 0;
- _BitScanReverse( &r, (unsigned long)val );
- return (unsigned)(r>>3);
-# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT)
- return (__builtin_clz((U32)val) >> 3);
-# else
- unsigned r;
- if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
- r += (!val);
- return r;
-# endif
- }
- }
-}
-
-static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
-{
- const BYTE* const pStart = pIn;
-
- while (likely(pIn compression run slower on incompressible data */
-
-
-/**************************************
-* Local Structures and types
-**************************************/
-typedef struct {
- U32 hashTable[HASH_SIZE_U32];
- U32 currentOffset;
- U32 initCheck;
- const BYTE* dictionary;
- BYTE* bufferStart; /* obsolete, used for slideInputBuffer */
- U32 dictSize;
-} LZ4_stream_t_internal;
-
-typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive;
-typedef enum { byPtr, byU32, byU16 } tableType_t;
-
-typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive;
-typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
-
-typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
-typedef enum { full = 0, partial = 1 } earlyEnd_directive;
-
-
-/**************************************
-* Local Utils
-**************************************/
-int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; }
-int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
-int LZ4_sizeofState() { return LZ4_STREAMSIZE; }
-
-
-
-/********************************
-* Compression functions
-********************************/
-
-static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType)
-{
- if (tableType == byU16)
- return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1)));
- else
- return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG));
-}
-
-static const U64 prime5bytes = 889523592379ULL;
-static U32 LZ4_hashSequence64(size_t sequence, tableType_t const tableType)
-{
- const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG;
- const U32 hashMask = (1<> (40 - hashLog)) & hashMask;
-}
-
-static U32 LZ4_hashSequenceT(size_t sequence, tableType_t const tableType)
-{
- if (LZ4_64bits())
- return LZ4_hashSequence64(sequence, tableType);
- return LZ4_hashSequence((U32)sequence, tableType);
-}
-
-static U32 LZ4_hashPosition(const void* p, tableType_t tableType) { return LZ4_hashSequenceT(LZ4_read_ARCH(p), tableType); }
-
-static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase)
-{
- switch (tableType)
- {
- case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; }
- case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; }
- case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; }
- }
-}
-
-static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
-{
- U32 h = LZ4_hashPosition(p, tableType);
- LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
-}
-
-static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase)
-{
- if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; }
- if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; }
- { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */
-}
-
-static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase)
-{
- U32 h = LZ4_hashPosition(p, tableType);
- return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
-}
-
-FORCE_INLINE int LZ4_compress_generic(
- void* const ctx,
- const char* const source,
- char* const dest,
- const int inputSize,
- const int maxOutputSize,
- const limitedOutput_directive outputLimited,
- const tableType_t tableType,
- const dict_directive dict,
- const dictIssue_directive dictIssue,
- const U32 acceleration)
-{
- LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx;
-
- const BYTE* ip = (const BYTE*) source;
- const BYTE* base;
- const BYTE* lowLimit;
- const BYTE* const lowRefLimit = ip - dictPtr->dictSize;
- const BYTE* const dictionary = dictPtr->dictionary;
- const BYTE* const dictEnd = dictionary + dictPtr->dictSize;
- const size_t dictDelta = dictEnd - (const BYTE*)source;
- const BYTE* anchor = (const BYTE*) source;
- const BYTE* const iend = ip + inputSize;
- const BYTE* const mflimit = iend - MFLIMIT;
- const BYTE* const matchlimit = iend - LASTLITERALS;
-
- BYTE* op = (BYTE*) dest;
- BYTE* const olimit = op + maxOutputSize;
-
- U32 forwardH;
- size_t refDelta=0;
-
- /* Init conditions */
- if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */
- switch(dict)
- {
- case noDict:
- default:
- base = (const BYTE*)source;
- lowLimit = (const BYTE*)source;
- break;
- case withPrefix64k:
- base = (const BYTE*)source - dictPtr->currentOffset;
- lowLimit = (const BYTE*)source - dictPtr->dictSize;
- break;
- case usingExtDict:
- base = (const BYTE*)source - dictPtr->currentOffset;
- lowLimit = (const BYTE*)source;
- break;
- }
- if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */
- if (inputSize> LZ4_skipTrigger);
-
- if (unlikely(forwardIp > mflimit)) goto _last_literals;
-
- match = LZ4_getPositionOnHash(h, ctx, tableType, base);
- if (dict==usingExtDict)
- {
- if (match<(const BYTE*)source)
- {
- refDelta = dictDelta;
- lowLimit = dictionary;
- }
- else
- {
- refDelta = 0;
- lowLimit = (const BYTE*)source;
- }
- }
- forwardH = LZ4_hashPosition(forwardIp, tableType);
- LZ4_putPositionOnHash(ip, h, ctx, tableType, base);
-
- } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0)
- || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
- || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) );
- }
-
- /* Catch up */
- while ((ip>anchor) && (match+refDelta > lowLimit) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; }
-
- {
- /* Encode Literal length */
- unsigned litLength = (unsigned)(ip - anchor);
- token = op++;
- if ((outputLimited) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit)))
- return 0; /* Check output limit */
- if (litLength>=RUN_MASK)
- {
- int len = (int)litLength-RUN_MASK;
- *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255;
- *op++ = (BYTE)len;
- }
- else *token = (BYTE)(litLength< matchlimit) limit = matchlimit;
- matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, limit);
- ip += MINMATCH + matchLength;
- if (ip==limit)
- {
- unsigned more = LZ4_count(ip, (const BYTE*)source, matchlimit);
- matchLength += more;
- ip += more;
- }
- }
- else
- {
- matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit);
- ip += MINMATCH + matchLength;
- }
-
- if ((outputLimited) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > olimit)))
- return 0; /* Check output limit */
- if (matchLength>=ML_MASK)
- {
- *token += ML_MASK;
- matchLength -= ML_MASK;
- for (; matchLength >= 510 ; matchLength-=510) { *op++ = 255; *op++ = 255; }
- if (matchLength >= 255) { matchLength-=255; *op++ = 255; }
- *op++ = (BYTE)matchLength;
- }
- else *token += (BYTE)(matchLength);
- }
-
- anchor = ip;
-
- /* Test end of chunk */
- if (ip > mflimit) break;
-
- /* Fill table */
- LZ4_putPosition(ip-2, ctx, tableType, base);
-
- /* Test next position */
- match = LZ4_getPosition(ip, ctx, tableType, base);
- if (dict==usingExtDict)
- {
- if (match<(const BYTE*)source)
- {
- refDelta = dictDelta;
- lowLimit = dictionary;
- }
- else
- {
- refDelta = 0;
- lowLimit = (const BYTE*)source;
- }
- }
- LZ4_putPosition(ip, ctx, tableType, base);
- if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1)
- && (match+MAX_DISTANCE>=ip)
- && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) )
- { token=op++; *token=0; goto _next_match; }
-
- /* Prepare next loop */
- forwardH = LZ4_hashPosition(++ip, tableType);
- }
-
-_last_literals:
- /* Encode Last Literals */
- {
- const size_t lastRun = (size_t)(iend - anchor);
- if ((outputLimited) && ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize))
- return 0; /* Check output limit */
- if (lastRun >= RUN_MASK)
- {
- size_t accumulator = lastRun - RUN_MASK;
- *op++ = RUN_MASK << ML_BITS;
- for(; accumulator >= 255 ; accumulator-=255) *op++ = 255;
- *op++ = (BYTE) accumulator;
- }
- else
- {
- *op++ = (BYTE)(lastRun<= LZ4_compressBound(inputSize))
- {
- if (inputSize < LZ4_64Klimit)
- return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
- else
- return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
- }
- else
- {
- if (inputSize < LZ4_64Klimit)
- return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
- else
- return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
- }
-}
-
-
-int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
-{
-#if (HEAPMODE)
- void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
-#else
- LZ4_stream_t ctx;
- void* ctxPtr = &ctx;
-#endif
-
- int result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration);
-
-#if (HEAPMODE)
- FREEMEM(ctxPtr);
-#endif
- return result;
-}
-
-
-int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize)
-{
- return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1);
-}
-
-
-/* hidden debug function */
-/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */
-int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
-{
- LZ4_stream_t ctx;
-
- LZ4_resetStream(&ctx);
-
- if (inputSize < LZ4_64Klimit)
- return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
- else
- return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration);
-}
-
-
-/********************************
-* destSize variant
-********************************/
-
-static int LZ4_compress_destSize_generic(
- void* const ctx,
- const char* const src,
- char* const dst,
- int* const srcSizePtr,
- const int targetDstSize,
- const tableType_t tableType)
-{
- const BYTE* ip = (const BYTE*) src;
- const BYTE* base = (const BYTE*) src;
- const BYTE* lowLimit = (const BYTE*) src;
- const BYTE* anchor = ip;
- const BYTE* const iend = ip + *srcSizePtr;
- const BYTE* const mflimit = iend - MFLIMIT;
- const BYTE* const matchlimit = iend - LASTLITERALS;
-
- BYTE* op = (BYTE*) dst;
- BYTE* const oend = op + targetDstSize;
- BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */;
- BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */);
- BYTE* const oMaxSeq = oMaxLit - 1 /* token */;
-
- U32 forwardH;
-
-
- /* Init conditions */
- if (targetDstSize < 1) return 0; /* Impossible to store anything */
- if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */
- if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */
- if (*srcSizePtr> LZ4_skipTrigger);
-
- if (unlikely(forwardIp > mflimit))
- goto _last_literals;
-
- match = LZ4_getPositionOnHash(h, ctx, tableType, base);
- forwardH = LZ4_hashPosition(forwardIp, tableType);
- LZ4_putPositionOnHash(ip, h, ctx, tableType, base);
-
- } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip))
- || (LZ4_read32(match) != LZ4_read32(ip)) );
- }
-
- /* Catch up */
- while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; }
-
- {
- /* Encode Literal length */
- unsigned litLength = (unsigned)(ip - anchor);
- token = op++;
- if (op + ((litLength+240)/255) + litLength > oMaxLit)
- {
- /* Not enough space for a last match */
- op--;
- goto _last_literals;
- }
- if (litLength>=RUN_MASK)
- {
- unsigned len = litLength - RUN_MASK;
- *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255;
- *op++ = (BYTE)len;
- }
- else *token = (BYTE)(litLength< oMaxMatch)
- {
- /* Match description too long : reduce it */
- matchLength = (15-1) + (oMaxMatch-op) * 255;
- }
- //printf("offset %5i, matchLength%5i \n", (int)(ip-match), matchLength + MINMATCH);
- ip += MINMATCH + matchLength;
-
- if (matchLength>=ML_MASK)
- {
- *token += ML_MASK;
- matchLength -= ML_MASK;
- while (matchLength >= 255) { matchLength-=255; *op++ = 255; }
- *op++ = (BYTE)matchLength;
- }
- else *token += (BYTE)(matchLength);
- }
-
- anchor = ip;
-
- /* Test end of block */
- if (ip > mflimit) break;
- if (op > oMaxSeq) break;
-
- /* Fill table */
- LZ4_putPosition(ip-2, ctx, tableType, base);
-
- /* Test next position */
- match = LZ4_getPosition(ip, ctx, tableType, base);
- LZ4_putPosition(ip, ctx, tableType, base);
- if ( (match+MAX_DISTANCE>=ip)
- && (LZ4_read32(match)==LZ4_read32(ip)) )
- { token=op++; *token=0; goto _next_match; }
-
- /* Prepare next loop */
- forwardH = LZ4_hashPosition(++ip, tableType);
- }
-
-_last_literals:
- /* Encode Last Literals */
- {
- size_t lastRunSize = (size_t)(iend - anchor);
- if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend)
- {
- /* adapt lastRunSize to fill 'dst' */
- lastRunSize = (oend-op) - 1;
- lastRunSize -= (lastRunSize+240)/255;
- }
- ip = anchor + lastRunSize;
-
- if (lastRunSize >= RUN_MASK)
- {
- size_t accumulator = lastRunSize - RUN_MASK;
- *op++ = RUN_MASK << ML_BITS;
- for(; accumulator >= 255 ; accumulator-=255) *op++ = 255;
- *op++ = (BYTE) accumulator;
- }
- else
- {
- *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) /* compression success is guaranteed */
- {
- return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1);
- }
- else
- {
- if (*srcSizePtr < LZ4_64Klimit)
- return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, byU16);
- else
- return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, LZ4_64bits() ? byU32 : byPtr);
- }
-}
-
-
-int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize)
-{
-#if (HEAPMODE)
- void* ctx = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */
-#else
- LZ4_stream_t ctxBody;
- void* ctx = &ctxBody;
-#endif
-
- int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize);
-
-#if (HEAPMODE)
- FREEMEM(ctx);
-#endif
- return result;
-}
-
-
-
-/********************************
-* Streaming functions
-********************************/
-
-LZ4_stream_t* LZ4_createStream(void)
-{
- LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64);
- LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */
- LZ4_resetStream(lz4s);
- return lz4s;
-}
-
-void LZ4_resetStream (LZ4_stream_t* LZ4_stream)
-{
- MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t));
-}
-
-int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
-{
- FREEMEM(LZ4_stream);
- return (0);
-}
-
-
-#define HASH_UNIT sizeof(size_t)
-int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
-{
- LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict;
- const BYTE* p = (const BYTE*)dictionary;
- const BYTE* const dictEnd = p + dictSize;
- const BYTE* base;
-
- if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */
- LZ4_resetStream(LZ4_dict);
-
- if (dictSize < (int)HASH_UNIT)
- {
- dict->dictionary = NULL;
- dict->dictSize = 0;
- return 0;
- }
-
- if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB;
- dict->currentOffset += 64 KB;
- base = p - dict->currentOffset;
- dict->dictionary = p;
- dict->dictSize = (U32)(dictEnd - p);
- dict->currentOffset += dict->dictSize;
-
- while (p <= dictEnd-HASH_UNIT)
- {
- LZ4_putPosition(p, dict->hashTable, byU32, base);
- p+=3;
- }
-
- return dict->dictSize;
-}
-
-
-static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src)
-{
- if ((LZ4_dict->currentOffset > 0x80000000) ||
- ((size_t)LZ4_dict->currentOffset > (size_t)src)) /* address space overflow */
- {
- /* rescale hash table */
- U32 delta = LZ4_dict->currentOffset - 64 KB;
- const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize;
- int i;
- for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0;
- else LZ4_dict->hashTable[i] -= delta;
- }
- LZ4_dict->currentOffset = 64 KB;
- if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB;
- LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize;
- }
-}
-
-
-int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
-{
- LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream;
- const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
-
- const BYTE* smallest = (const BYTE*) source;
- if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */
- if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd;
- LZ4_renormDictT(streamPtr, smallest);
- if (acceleration < 1) acceleration = ACCELERATION_DEFAULT;
-
- /* Check overlapping input/dictionary space */
- {
- const BYTE* sourceEnd = (const BYTE*) source + inputSize;
- if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd))
- {
- streamPtr->dictSize = (U32)(dictEnd - sourceEnd);
- if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB;
- if (streamPtr->dictSize < 4) streamPtr->dictSize = 0;
- streamPtr->dictionary = dictEnd - streamPtr->dictSize;
- }
- }
-
- /* prefix mode : source data follows dictionary */
- if (dictEnd == (const BYTE*)source)
- {
- int result;
- if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
- result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration);
- else
- result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration);
- streamPtr->dictSize += (U32)inputSize;
- streamPtr->currentOffset += (U32)inputSize;
- return result;
- }
-
- /* external dictionary mode */
- {
- int result;
- if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset))
- result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration);
- else
- result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration);
- streamPtr->dictionary = (const BYTE*)source;
- streamPtr->dictSize = (U32)inputSize;
- streamPtr->currentOffset += (U32)inputSize;
- return result;
- }
-}
-
-
-/* Hidden debug function, to force external dictionary mode */
-int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize)
-{
- LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict;
- int result;
- const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize;
-
- const BYTE* smallest = dictEnd;
- if (smallest > (const BYTE*) source) smallest = (const BYTE*) source;
- LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest);
-
- result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1);
-
- streamPtr->dictionary = (const BYTE*)source;
- streamPtr->dictSize = (U32)inputSize;
- streamPtr->currentOffset += (U32)inputSize;
-
- return result;
-}
-
-
-int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize)
-{
- LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict;
- const BYTE* previousDictEnd = dict->dictionary + dict->dictSize;
-
- if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */
- if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize;
-
- memmove(safeBuffer, previousDictEnd - dictSize, dictSize);
-
- dict->dictionary = (const BYTE*)safeBuffer;
- dict->dictSize = (U32)dictSize;
-
- return dictSize;
-}
-
-
-
-/*******************************
-* Decompression functions
-*******************************/
-/*
- * This generic decompression function cover all use cases.
- * It shall be instantiated several times, using different sets of directives
- * Note that it is essential this generic function is really inlined,
- * in order to remove useless branches during compilation optimization.
- */
-FORCE_INLINE int LZ4_decompress_generic(
- const char* const source,
- char* const dest,
- int inputSize,
- int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */
-
- int endOnInput, /* endOnOutputSize, endOnInputSize */
- int partialDecoding, /* full, partial */
- int targetOutputSize, /* only used if partialDecoding==partial */
- int dict, /* noDict, withPrefix64k, usingExtDict */
- const BYTE* const lowPrefix, /* == dest if dict == noDict */
- const BYTE* const dictStart, /* only if dict==usingExtDict */
- const size_t dictSize /* note : = 0 if noDict */
- )
-{
- /* Local Variables */
- const BYTE* ip = (const BYTE*) source;
- const BYTE* const iend = ip + inputSize;
-
- BYTE* op = (BYTE*) dest;
- BYTE* const oend = op + outputSize;
- BYTE* cpy;
- BYTE* oexit = op + targetOutputSize;
- const BYTE* const lowLimit = lowPrefix - dictSize;
-
- const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize;
- const size_t dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4};
- const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3};
-
- const int safeDecode = (endOnInput==endOnInputSize);
- const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB)));
-
-
- /* Special cases */
- if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */
- if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */
- if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1);
-
-
- /* Main Loop */
- while (1)
- {
- unsigned token;
- size_t length;
- const BYTE* match;
-
- /* get literal length */
- token = *ip++;
- if ((length=(token>>ML_BITS)) == RUN_MASK)
- {
- unsigned s;
- do
- {
- s = *ip++;
- length += s;
- }
- while (likely((endOnInput)?ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) )
- || ((!endOnInput) && (cpy>oend-COPYLENGTH)))
- {
- if (partialDecoding)
- {
- if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */
- if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */
- }
- else
- {
- if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */
- if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */
- }
- memcpy(op, ip, length);
- ip += length;
- op += length;
- break; /* Necessarily EOF, due to parsing restrictions */
- }
- LZ4_wildCopy(op, ip, cpy);
- ip += length; op = cpy;
-
- /* get offset */
- match = cpy - LZ4_readLE16(ip); ip+=2;
- if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside destination buffer */
-
- /* get matchlength */
- length = token & ML_MASK;
- if (length == ML_MASK)
- {
- unsigned s;
- do
- {
- if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error;
- s = *ip++;
- length += s;
- } while (s==255);
- if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */
- }
- length += MINMATCH;
-
- /* check external dictionary */
- if ((dict==usingExtDict) && (match < lowPrefix))
- {
- if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */
-
- if (length <= (size_t)(lowPrefix-match))
- {
- /* match can be copied as a single segment from external dictionary */
- match = dictEnd - (lowPrefix-match);
- memmove(op, match, length); op += length;
- }
- else
- {
- /* match encompass external dictionary and current segment */
- size_t copySize = (size_t)(lowPrefix-match);
- memcpy(op, dictEnd - copySize, copySize);
- op += copySize;
- copySize = length - copySize;
- if (copySize > (size_t)(op-lowPrefix)) /* overlap within current segment */
- {
- BYTE* const endOfMatch = op + copySize;
- const BYTE* copyFrom = lowPrefix;
- while (op < endOfMatch) *op++ = *copyFrom++;
- }
- else
- {
- memcpy(op, lowPrefix, copySize);
- op += copySize;
- }
- }
- continue;
- }
-
- /* copy repeated sequence */
- cpy = op + length;
- if (unlikely((op-match)<8))
- {
- const size_t dec64 = dec64table[op-match];
- op[0] = match[0];
- op[1] = match[1];
- op[2] = match[2];
- op[3] = match[3];
- match += dec32table[op-match];
- LZ4_copy4(op+4, match);
- op += 8; match -= dec64;
- } else { LZ4_copy8(op, match); op+=8; match+=8; }
-
- if (unlikely(cpy>oend-12))
- {
- if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals */
- if (op < oend-8)
- {
- LZ4_wildCopy(op, match, oend-8);
- match += (oend-8) - op;
- op = oend-8;
- }
- while (opprefixSize = (size_t) dictSize;
- lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize;
- lz4sd->externalDict = NULL;
- lz4sd->extDictSize = 0;
- return 1;
-}
-
-/*
-*_continue() :
- These decoding functions allow decompression of multiple blocks in "streaming" mode.
- Previously decoded blocks must still be available at the memory position where they were decoded.
- If it's not possible, save the relevant part of decoded data into a safe buffer,
- and indicate where it stands using LZ4_setStreamDecode()
-*/
-int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)
-{
- LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
- int result;
-
- if (lz4sd->prefixEnd == (BYTE*)dest)
- {
- result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
- endOnInputSize, full, 0,
- usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
- if (result <= 0) return result;
- lz4sd->prefixSize += result;
- lz4sd->prefixEnd += result;
- }
- else
- {
- lz4sd->extDictSize = lz4sd->prefixSize;
- lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
- result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize,
- endOnInputSize, full, 0,
- usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize);
- if (result <= 0) return result;
- lz4sd->prefixSize = result;
- lz4sd->prefixEnd = (BYTE*)dest + result;
- }
-
- return result;
-}
-
-int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize)
-{
- LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode;
- int result;
-
- if (lz4sd->prefixEnd == (BYTE*)dest)
- {
- result = LZ4_decompress_generic(source, dest, 0, originalSize,
- endOnOutputSize, full, 0,
- usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize);
- if (result <= 0) return result;
- lz4sd->prefixSize += originalSize;
- lz4sd->prefixEnd += originalSize;
- }
- else
- {
- lz4sd->extDictSize = lz4sd->prefixSize;
- lz4sd->externalDict = (BYTE*)dest - lz4sd->extDictSize;
- result = LZ4_decompress_generic(source, dest, 0, originalSize,
- endOnOutputSize, full, 0,
- usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize);
- if (result <= 0) return result;
- lz4sd->prefixSize = originalSize;
- lz4sd->prefixEnd = (BYTE*)dest + originalSize;
- }
-
- return result;
-}
-
-
-/*
-Advanced decoding functions :
-*_usingDict() :
- These decoding functions work the same as "_continue" ones,
- the dictionary must be explicitly provided within parameters
-*/
-
-FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize)
-{
- if (dictSize==0)
- return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0);
- if (dictStart+dictSize == dest)
- {
- if (dictSize >= (int)(64 KB - 1))
- return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0);
- return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0);
- }
- return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
-}
-
-int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
-{
- return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize);
-}
-
-int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize)
-{
- return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize);
-}
-
-/* debug function */
-int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
-{
- return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
-}
-
-
-/***************************************************
-* Obsolete Functions
-***************************************************/
-/* obsolete compression functions */
-int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); }
-int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); }
-int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); }
-int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); }
-int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); }
-int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); }
-
-/*
-These function names are deprecated and should no longer be used.
-They are only provided here for compatibility with older user programs.
-- LZ4_uncompress is totally equivalent to LZ4_decompress_fast
-- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe
-*/
-int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); }
-int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); }
-
-
-/* Obsolete Streaming functions */
-
-int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; }
-
-static void LZ4_init(LZ4_stream_t_internal* lz4ds, BYTE* base)
-{
- MEM_INIT(lz4ds, 0, LZ4_STREAMSIZE);
- lz4ds->bufferStart = base;
-}
-
-int LZ4_resetStreamState(void* state, char* inputBuffer)
-{
- if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */
- LZ4_init((LZ4_stream_t_internal*)state, (BYTE*)inputBuffer);
- return 0;
-}
-
-void* LZ4_create (char* inputBuffer)
-{
- void* lz4ds = ALLOCATOR(8, LZ4_STREAMSIZE_U64);
- LZ4_init ((LZ4_stream_t_internal*)lz4ds, (BYTE*)inputBuffer);
- return lz4ds;
-}
-
-char* LZ4_slideInputBuffer (void* LZ4_Data)
-{
- LZ4_stream_t_internal* ctx = (LZ4_stream_t_internal*)LZ4_Data;
- int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB);
- return (char*)(ctx->bufferStart + dictSize);
-}
-
-/* Obsolete streaming decompression functions */
-
-int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize)
-{
- return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB);
-}
-
-int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize)
-{
- return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB);
-}
-
-#endif /* LZ4_COMMONDEFS_ONLY */
-
diff --git a/core-writer/external/bitshuffle/lz4.h b/core-writer/external/bitshuffle/lz4.h
deleted file mode 100644
index 3e74002..0000000
--- a/core-writer/external/bitshuffle/lz4.h
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- LZ4 - Fast LZ compression algorithm
- Header File
- Copyright (C) 2011-2015, Yann Collet.
-
- BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- You can contact the author at :
- - LZ4 source repository : https://github.com/Cyan4973/lz4
- - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
-*/
-#pragma once
-
-#if defined (__cplusplus)
-extern "C" {
-#endif
-
-/*
- * lz4.h provides block compression functions, and gives full buffer control to programmer.
- * If you need to generate inter-operable compressed data (respecting LZ4 frame specification),
- * and can let the library handle its own memory, please use lz4frame.h instead.
-*/
-
-/**************************************
-* Version
-**************************************/
-#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
-#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */
-#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */
-#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
-int LZ4_versionNumber (void);
-
-/**************************************
-* Tuning parameter
-**************************************/
-/*
- * LZ4_MEMORY_USAGE :
- * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
- * Increasing memory usage improves compression ratio
- * Reduced memory usage can improve speed, due to cache effect
- * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
- */
-#define LZ4_MEMORY_USAGE 14
-
-
-/**************************************
-* Simple Functions
-**************************************/
-
-int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
-int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
-
-/*
-LZ4_compress_default() :
- Compresses 'sourceSize' bytes from buffer 'source'
- into already allocated 'dest' buffer of size 'maxDestSize'.
- Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize).
- It also runs faster, so it's a recommended setting.
- If the function cannot compress 'source' into a more limited 'dest' budget,
- compression stops *immediately*, and the function result is zero.
- As a consequence, 'dest' content is not valid.
- This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
- sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE
- maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
- return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
- or 0 if compression fails
-
-LZ4_decompress_safe() :
- compressedSize : is the precise full size of the compressed block.
- maxDecompressedSize : is the size of destination buffer, which must be already allocated.
- return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
- If destination buffer is not large enough, decoding will stop and output an error code (<0).
- If the source stream is detected malformed, the function will stop decoding and return a negative result.
- This function is protected against buffer overflow exploits, including malicious data packets.
- It never writes outside output buffer, nor reads outside input buffer.
-*/
-
-
-/**************************************
-* Advanced Functions
-**************************************/
-#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
-#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
-
-/*
-LZ4_compressBound() :
- Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
- This function is primarily useful for memory allocation purposes (destination buffer size).
- Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
- Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize)
- inputSize : max supported value is LZ4_MAX_INPUT_SIZE
- return : maximum output size in a "worst case" scenario
- or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
-*/
-int LZ4_compressBound(int inputSize);
-
-/*
-LZ4_compress_fast() :
- Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
- The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
- It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
- An acceleration value of "1" is the same as regular LZ4_compress_default()
- Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
-*/
-int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
-
-
-/*
-LZ4_compress_fast_extState() :
- Same compression function, just using an externally allocated memory space to store compression state.
- Use LZ4_sizeofState() to know how much memory must be allocated,
- and allocate it on 8-bytes boundaries (using malloc() typically).
- Then, provide it as 'void* state' to compression function.
-*/
-int LZ4_sizeofState(void);
-int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration);
-
-
-/*
-LZ4_compress_destSize() :
- Reverse the logic, by compressing as much data as possible from 'source' buffer
- into already allocated buffer 'dest' of size 'targetDestSize'.
- This function either compresses the entire 'source' content into 'dest' if it's large enough,
- or fill 'dest' buffer completely with as much data as possible from 'source'.
- *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'.
- New value is necessarily <= old value.
- return : Nb bytes written into 'dest' (necessarily <= targetDestSize)
- or 0 if compression fails
-*/
-int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize);
-
-
-/*
-LZ4_decompress_fast() :
- originalSize : is the original and therefore uncompressed size
- return : the number of bytes read from the source buffer (in other words, the compressed size)
- If the source stream is detected malformed, the function will stop decoding and return a negative result.
- Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
- note : This function fully respect memory boundaries for properly formed compressed data.
- It is a bit faster than LZ4_decompress_safe().
- However, it does not provide any protection against intentionally modified data stream (malicious input).
- Use this function in trusted environment only (data to decode comes from a trusted source).
-*/
-int LZ4_decompress_fast (const char* source, char* dest, int originalSize);
-
-/*
-LZ4_decompress_safe_partial() :
- This function decompress a compressed block of size 'compressedSize' at position 'source'
- into destination buffer 'dest' of size 'maxDecompressedSize'.
- The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
- reducing decompression time.
- return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
- Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
- Always control how many bytes were decoded.
- If the source stream is detected malformed, the function will stop decoding and return a negative result.
- This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
-*/
-int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
-
-
-/***********************************************
-* Streaming Compression Functions
-***********************************************/
-#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
-#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long))
-/*
- * LZ4_stream_t
- * information structure to track an LZ4 stream.
- * important : init this structure content before first use !
- * note : only allocated directly the structure if you are statically linking LZ4
- * If you are using liblz4 as a DLL, please use below construction methods instead.
- */
-typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t;
-
-/*
- * LZ4_resetStream
- * Use this function to init an allocated LZ4_stream_t structure
- */
-void LZ4_resetStream (LZ4_stream_t* streamPtr);
-
-/*
- * LZ4_createStream will allocate and initialize an LZ4_stream_t structure
- * LZ4_freeStream releases its memory.
- * In the context of a DLL (liblz4), please use these methods rather than the static struct.
- * They are more future proof, in case of a change of LZ4_stream_t size.
- */
-LZ4_stream_t* LZ4_createStream(void);
-int LZ4_freeStream (LZ4_stream_t* streamPtr);
-
-/*
- * LZ4_loadDict
- * Use this function to load a static dictionary into LZ4_stream.
- * Any previous data will be forgotten, only 'dictionary' will remain in memory.
- * Loading a size of 0 is allowed.
- * Return : dictionary size, in bytes (necessarily <= 64 KB)
- */
-int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
-
-/*
- * LZ4_compress_fast_continue
- * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
- * Important : Previous data blocks are assumed to still be present and unmodified !
- * 'dst' buffer must be already allocated.
- * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
- * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
- */
-int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
-
-/*
- * LZ4_saveDict
- * If previously compressed data block is not guaranteed to remain available at its memory location
- * save it into a safer place (char* safeBuffer)
- * Note : you don't need to call LZ4_loadDict() afterwards,
- * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue()
- * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error
- */
-int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
-
-
-/************************************************
-* Streaming Decompression Functions
-************************************************/
-
-#define LZ4_STREAMDECODESIZE_U64 4
-#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
-typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t;
-/*
- * LZ4_streamDecode_t
- * information structure to track an LZ4 stream.
- * init this structure content using LZ4_setStreamDecode or memset() before first use !
- *
- * In the context of a DLL (liblz4) please prefer usage of construction methods below.
- * They are more future proof, in case of a change of LZ4_streamDecode_t size in the future.
- * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure
- * LZ4_freeStreamDecode releases its memory.
- */
-LZ4_streamDecode_t* LZ4_createStreamDecode(void);
-int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
-
-/*
- * LZ4_setStreamDecode
- * Use this function to instruct where to find the dictionary.
- * Setting a size of 0 is allowed (same effect as reset).
- * Return : 1 if OK, 0 if error
- */
-int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
-
-/*
-*_continue() :
- These decoding functions allow decompression of multiple blocks in "streaming" mode.
- Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
- In the case of a ring buffers, decoding buffer must be either :
- - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
- In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
- maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
- In which case, encoding and decoding buffers do not need to be synchronized,
- and encoding ring buffer can have any size, including small ones ( < 64 KB).
- - _At least_ 64 KB + 8 bytes + maxBlockSize.
- In which case, encoding and decoding buffers do not need to be synchronized,
- and encoding ring buffer can have any size, including larger than decoding buffer.
- Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
- and indicate where it is saved using LZ4_setStreamDecode()
-*/
-int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
-int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
-
-
-/*
-Advanced decoding functions :
-*_usingDict() :
- These decoding functions work the same as
- a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue()
- They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure.
-*/
-int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
-int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
-
-
-
-/**************************************
-* Obsolete Functions
-**************************************/
-/* Deprecate Warnings */
-/* Should these warnings messages be a problem,
- it is generally possible to disable them,
- with -Wno-deprecated-declarations for gcc
- or _CRT_SECURE_NO_WARNINGS in Visual for example.
- You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */
-#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK
-# define LZ4_DEPRECATE_WARNING_DEFBLOCK
-# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-# if (LZ4_GCC_VERSION >= 405) || defined(__clang__)
-# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
-# elif (LZ4_GCC_VERSION >= 301)
-# define LZ4_DEPRECATED(message) __attribute__((deprecated))
-# elif defined(_MSC_VER)
-# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
-# else
-# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
-# define LZ4_DEPRECATED(message)
-# endif
-#endif /* LZ4_DEPRECATE_WARNING_DEFBLOCK */
-
-/* Obsolete compression functions */
-/* These functions are planned to start generate warnings by r131 approximately */
-int LZ4_compress (const char* source, char* dest, int sourceSize);
-int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
-int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
-int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
-int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
-int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
-
-/* Obsolete decompression functions */
-/* These function names are completely deprecated and must no longer be used.
- They are only provided here for compatibility with older programs.
- - LZ4_uncompress is the same as LZ4_decompress_fast
- - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
- These function prototypes are now disabled; uncomment them only if you really need them.
- It is highly recommended to stop using these prototypes and migrate to maintained ones */
-/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */
-/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */
-
-/* Obsolete streaming functions; use new streaming interface whenever possible */
-LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void);
-LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer);
-LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state);
-
-/* Obsolete streaming decoding functions */
-LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
-LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
-
-
-#if defined (__cplusplus)
-}
-#endif
diff --git a/core-writer/external/crow_all.h b/core-writer/external/crow_all.h
deleted file mode 100644
index 72cec04..0000000
--- a/core-writer/external/crow_all.h
+++ /dev/null
@@ -1,9787 +0,0 @@
-#pragma once
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-
-namespace crow
-{
-// ----------------------------------------------------------------------------
-// qs_parse (modified)
-// https://github.com/bartgrantham/qs_parse
-// ----------------------------------------------------------------------------
-/* Similar to strncmp, but handles URL-encoding for either string */
-int qs_strncmp(const char * s, const char * qs, size_t n);
-
-
-/* Finds the beginning of each key/value pair and stores a pointer in qs_kv.
- * Also decodes the value portion of the k/v pair *in-place*. In a future
- * enhancement it will also have a compile-time option of sorting qs_kv
- * alphabetically by key. */
-int qs_parse(char * qs, char * qs_kv[], int qs_kv_size);
-
-
-/* Used by qs_parse to decode the value portion of a k/v pair */
-int qs_decode(char * qs);
-
-
-/* Looks up the value according to the key on a pre-processed query string
- * A future enhancement will be a compile-time option to look up the key
- * in a pre-sorted qs_kv array via a binary search. */
-//char * qs_k2v(const char * key, char * qs_kv[], int qs_kv_size);
- char * qs_k2v(const char * key, char * const * qs_kv, int qs_kv_size, int nth);
-
-
-/* Non-destructive lookup of value, based on key. User provides the
- * destinaton string and length. */
-char * qs_scanvalue(const char * key, const char * qs, char * val, size_t val_len);
-
-// TODO: implement sorting of the qs_kv array; for now ensure it's not compiled
-#undef _qsSORTING
-
-// isxdigit _is_ available in , but let's avoid another header instead
-#define CROW_QS_ISHEX(x) ((((x)>='0'&&(x)<='9') || ((x)>='A'&&(x)<='F') || ((x)>='a'&&(x)<='f')) ? 1 : 0)
-#define CROW_QS_HEX2DEC(x) (((x)>='0'&&(x)<='9') ? (x)-48 : ((x)>='A'&&(x)<='F') ? (x)-55 : ((x)>='a'&&(x)<='f') ? (x)-87 : 0)
-#define CROW_QS_ISQSCHR(x) ((((x)=='=')||((x)=='#')||((x)=='&')||((x)=='\0')) ? 0 : 1)
-
-inline int qs_strncmp(const char * s, const char * qs, size_t n)
-{
- int i=0;
- unsigned char u1, u2, unyb, lnyb;
-
- while(n-- > 0)
- {
- u1 = (unsigned char) *s++;
- u2 = (unsigned char) *qs++;
-
- if ( ! CROW_QS_ISQSCHR(u1) ) { u1 = '\0'; }
- if ( ! CROW_QS_ISQSCHR(u2) ) { u2 = '\0'; }
-
- if ( u1 == '+' ) { u1 = ' '; }
- if ( u1 == '%' ) // easier/safer than scanf
- {
- unyb = (unsigned char) *s++;
- lnyb = (unsigned char) *s++;
- if ( CROW_QS_ISHEX(unyb) && CROW_QS_ISHEX(lnyb) )
- u1 = (CROW_QS_HEX2DEC(unyb) * 16) + CROW_QS_HEX2DEC(lnyb);
- else
- u1 = '\0';
- }
-
- if ( u2 == '+' ) { u2 = ' '; }
- if ( u2 == '%' ) // easier/safer than scanf
- {
- unyb = (unsigned char) *qs++;
- lnyb = (unsigned char) *qs++;
- if ( CROW_QS_ISHEX(unyb) && CROW_QS_ISHEX(lnyb) )
- u2 = (CROW_QS_HEX2DEC(unyb) * 16) + CROW_QS_HEX2DEC(lnyb);
- else
- u2 = '\0';
- }
-
- if ( u1 != u2 )
- return u1 - u2;
- if ( u1 == '\0' )
- return 0;
- i++;
- }
- if ( CROW_QS_ISQSCHR(*qs) )
- return -1;
- else
- return 0;
-}
-
-
-inline int qs_parse(char * qs, char * qs_kv[], int qs_kv_size)
-{
- int i, j;
- char * substr_ptr;
-
- for(i=0; i means x iterations of this loop -> means *x+1* k/v pairs
-
- // we only decode the values in place, the keys could have '='s in them
- // which will hose our ability to distinguish keys from values later
- for(j=0; j> qs_dict_name2kv(const char * dict_name, char * const * qs_kv, int qs_kv_size, int nth = 0)
-{
- int i;
- size_t name_len, skip_to_eq, skip_to_brace_open, skip_to_brace_close;
-
- name_len = strlen(dict_name);
-
-#ifdef _qsSORTING
-// TODO: binary search for key in the sorted qs_kv
-#else // _qsSORTING
- for(i=0; i 0 &&
- skip_to_brace_close > 0 &&
- nth == 0 )
- {
- auto key = std::string(qs_kv[i] + skip_to_brace_open, skip_to_brace_close - skip_to_brace_open);
- auto value = std::string(qs_kv[i] + skip_to_eq);
- return boost::make_optional(std::make_pair(key, value));
- }
- else
- {
- --nth;
- }
- }
- }
-#endif // _qsSORTING
-
- return boost::none;
-}
-
-
-inline char * qs_scanvalue(const char * key, const char * qs, char * val, size_t val_len)
-{
- size_t i, key_len;
- const char * tmp;
-
- // find the beginning of the k/v substrings
- if ( (tmp = strchr(qs, '?')) != NULL )
- qs = tmp + 1;
-
- key_len = strlen(key);
- while(qs[0] != '#' && qs[0] != '\0')
- {
- if ( qs_strncmp(key, qs, key_len) == 0 )
- break;
- qs += strcspn(qs, "&") + 1;
- }
-
- if ( qs[0] == '\0' ) return NULL;
-
- qs += strcspn(qs, "=&#");
- if ( qs[0] == '=' )
- {
- qs++;
- i = strcspn(qs, "&=#");
-#ifdef _MSC_VER
- strncpy_s(val, val_len, qs, (val_len - 1)<(i + 1) ? (val_len - 1) : (i + 1));
-#else
- strncpy(val, qs, (val_len - 1)<(i + 1) ? (val_len - 1) : (i + 1));
-#endif
- qs_decode(val);
- }
- else
- {
- if ( val_len > 0 )
- val[0] = '\0';
- }
-
- return val;
-}
-}
-// ----------------------------------------------------------------------------
-
-
-namespace crow
-{
- class query_string
- {
- public:
- static const int MAX_KEY_VALUE_PAIRS_COUNT = 256;
-
- query_string()
- {
-
- }
-
- query_string(const query_string& qs)
- : url_(qs.url_)
- {
- for(auto p:qs.key_value_pairs_)
- {
- key_value_pairs_.push_back((char*)(p-qs.url_.c_str()+url_.c_str()));
- }
- }
-
- query_string& operator = (const query_string& qs)
- {
- url_ = qs.url_;
- key_value_pairs_.clear();
- for(auto p:qs.key_value_pairs_)
- {
- key_value_pairs_.push_back((char*)(p-qs.url_.c_str()+url_.c_str()));
- }
- return *this;
- }
-
- query_string& operator = (query_string&& qs)
- {
- key_value_pairs_ = std::move(qs.key_value_pairs_);
- char* old_data = (char*)qs.url_.c_str();
- url_ = std::move(qs.url_);
- for(auto& p:key_value_pairs_)
- {
- p += (char*)url_.c_str() - old_data;
- }
- return *this;
- }
-
-
- query_string(std::string url)
- : url_(std::move(url))
- {
- if (url_.empty())
- return;
-
- key_value_pairs_.resize(MAX_KEY_VALUE_PAIRS_COUNT);
-
- int count = qs_parse(&url_[0], &key_value_pairs_[0], MAX_KEY_VALUE_PAIRS_COUNT);
- key_value_pairs_.resize(count);
- }
-
- void clear()
- {
- key_value_pairs_.clear();
- url_.clear();
- }
-
- friend std::ostream& operator<<(std::ostream& os, const query_string& qs)
- {
- os << "[ ";
- for(size_t i = 0; i < qs.key_value_pairs_.size(); ++i) {
- if (i)
- os << ", ";
- os << qs.key_value_pairs_[i];
- }
- os << " ]";
- return os;
-
- }
-
- char* get (const std::string& name) const
- {
- char* ret = qs_k2v(name.c_str(), key_value_pairs_.data(), key_value_pairs_.size());
- return ret;
- }
-
- std::vector get_list (const std::string& name) const
- {
- std::vector ret;
- std::string plus = name + "[]";
- char* element = nullptr;
-
- int count = 0;
- while(1)
- {
- element = qs_k2v(plus.c_str(), key_value_pairs_.data(), key_value_pairs_.size(), count++);
- if (!element)
- break;
- ret.push_back(element);
- }
- return ret;
- }
-
- std::unordered_map get_dict (const std::string& name) const
- {
- std::unordered_map ret;
-
- int count = 0;
- while(1)
- {
- if (auto element = qs_dict_name2kv(name.c_str(), key_value_pairs_.data(), key_value_pairs_.size(), count++))
- ret.insert(*element);
- else
- break;
- }
- return ret;
- }
-
- private:
- std::string url_;
- std::vector key_value_pairs_;
- };
-
-} // end namespace
-
-
-
-/* merged revision: 5b951d74bd66ec9d38448e0a85b1cf8b85d97db3 */
-/* Copyright Joyent, Inc. and other Node contributors. All rights reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-#ifndef CROW_http_parser_h
-#define CROW_http_parser_h
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* Also update SONAME in the Makefile whenever you change these. */
-#define CROW_HTTP_PARSER_VERSION_MAJOR 2
-#define CROW_HTTP_PARSER_VERSION_MINOR 3
-#define CROW_HTTP_PARSER_VERSION_PATCH 0
-
-#include
-#if defined(_WIN32) && !defined(__MINGW32__) && (!defined(_MSC_VER) || _MSC_VER<1600)
-#include
-#include
-typedef __int8 int8_t;
-typedef unsigned __int8 uint8_t;
-typedef __int16 int16_t;
-typedef unsigned __int16 uint16_t;
-typedef __int32 int32_t;
-typedef unsigned __int32 uint32_t;
-typedef __int64 int64_t;
-typedef unsigned __int64 uint64_t;
-#else
-#include
-#endif
-
-/* Compile with -DHTTP_PARSER_STRICT=0 to make less checks, but run
- * faster
- */
-#ifndef CROW_HTTP_PARSER_STRICT
-# define CROW_HTTP_PARSER_STRICT 1
-#endif
-
-/* Maximium header size allowed. If the macro is not defined
- * before including this header then the default is used. To
- * change the maximum header size, define the macro in the build
- * environment (e.g. -DHTTP_MAX_HEADER_SIZE=). To remove
- * the effective limit on the size of the header, define the macro
- * to a very large number (e.g. -DHTTP_MAX_HEADER_SIZE=0x7fffffff)
- */
-#ifndef CROW_HTTP_MAX_HEADER_SIZE
-# define CROW_HTTP_MAX_HEADER_SIZE (80*1024)
-#endif
-
-typedef struct http_parser http_parser;
-typedef struct http_parser_settings http_parser_settings;
-
-
-/* Callbacks should return non-zero to indicate an error. The parser will
- * then halt execution.
- *
- * The one exception is on_headers_complete. In a HTTP_RESPONSE parser
- * returning '1' from on_headers_complete will tell the parser that it
- * should not expect a body. This is used when receiving a response to a
- * HEAD request which may contain 'Content-Length' or 'Transfer-Encoding:
- * chunked' headers that indicate the presence of a body.
- *
- * http_data_cb does not return data chunks. It will be call arbitrarally
- * many times for each string. E.G. you might get 10 callbacks for "on_url"
- * each providing just a few characters more data.
- */
-typedef int (*http_data_cb) (http_parser*, const char *at, size_t length);
-typedef int (*http_cb) (http_parser*);
-
-
-/* Request Methods */
-#define CROW_HTTP_METHOD_MAP(CROW_XX) \
- CROW_XX(0, DELETE, DELETE) \
- CROW_XX(1, GET, GET) \
- CROW_XX(2, HEAD, HEAD) \
- CROW_XX(3, POST, POST) \
- CROW_XX(4, PUT, PUT) \
- /* pathological */ \
- CROW_XX(5, CONNECT, CONNECT) \
- CROW_XX(6, OPTIONS, OPTIONS) \
- CROW_XX(7, TRACE, TRACE) \
- /* webdav */ \
- CROW_XX(8, COPY, COPY) \
- CROW_XX(9, LOCK, LOCK) \
- CROW_XX(10, MKCOL, MKCOL) \
- CROW_XX(11, MOVE, MOVE) \
- CROW_XX(12, PROPFIND, PROPFIND) \
- CROW_XX(13, PROPPATCH, PROPPATCH) \
- CROW_XX(14, SEARCH, SEARCH) \
- CROW_XX(15, UNLOCK, UNLOCK) \
- /* subversion */ \
- CROW_XX(16, REPORT, REPORT) \
- CROW_XX(17, MKACTIVITY, MKACTIVITY) \
- CROW_XX(18, CHECKOUT, CHECKOUT) \
- CROW_XX(19, MERGE, MERGE) \
- /* upnp */ \
- CROW_XX(20, MSEARCH, M-SEARCH) \
- CROW_XX(21, NOTIFY, NOTIFY) \
- CROW_XX(22, SUBSCRIBE, SUBSCRIBE) \
- CROW_XX(23, UNSUBSCRIBE, UNSUBSCRIBE) \
- /* RFC-5789 */ \
- CROW_XX(24, PATCH, PATCH) \
- CROW_XX(25, PURGE, PURGE) \
- /* CalDAV */ \
- CROW_XX(26, MKCALENDAR, MKCALENDAR) \
-
-enum http_method
- {
-#define CROW_XX(num, name, string) HTTP_##name = num,
- CROW_HTTP_METHOD_MAP(CROW_XX)
-#undef CROW_XX
- };
-
-
-enum http_parser_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH };
-
-
-/* Flag values for http_parser.flags field */
-enum flags
- { F_CHUNKED = 1 << 0
- , F_CONNECTION_KEEP_ALIVE = 1 << 1
- , F_CONNECTION_CLOSE = 1 << 2
- , F_TRAILING = 1 << 3
- , F_UPGRADE = 1 << 4
- , F_SKIPBODY = 1 << 5
- };
-
-
-/* Map for errno-related constants
- *
- * The provided argument should be a macro that takes 2 arguments.
- */
-#define CROW_HTTP_ERRNO_MAP(CROW_XX) \
- /* No error */ \
- CROW_XX(OK, "success") \
- \
- /* Callback-related errors */ \
- CROW_XX(CB_message_begin, "the on_message_begin callback failed") \
- CROW_XX(CB_url, "the on_url callback failed") \
- CROW_XX(CB_header_field, "the on_header_field callback failed") \
- CROW_XX(CB_header_value, "the on_header_value callback failed") \
- CROW_XX(CB_headers_complete, "the on_headers_complete callback failed") \
- CROW_XX(CB_body, "the on_body callback failed") \
- CROW_XX(CB_message_complete, "the on_message_complete callback failed") \
- CROW_XX(CB_status, "the on_status callback failed") \
- \
- /* Parsing-related errors */ \
- CROW_XX(INVALID_EOF_STATE, "stream ended at an unexpected time") \
- CROW_XX(HEADER_OVERFLOW, \
- "too many header bytes seen; overflow detected") \
- CROW_XX(CLOSED_CONNECTION, \
- "data received after completed connection: close message") \
- CROW_XX(INVALID_VERSION, "invalid HTTP version") \
- CROW_XX(INVALID_STATUS, "invalid HTTP status code") \
- CROW_XX(INVALID_METHOD, "invalid HTTP method") \
- CROW_XX(INVALID_URL, "invalid URL") \
- CROW_XX(INVALID_HOST, "invalid host") \
- CROW_XX(INVALID_PORT, "invalid port") \
- CROW_XX(INVALID_PATH, "invalid path") \
- CROW_XX(INVALID_QUERY_STRING, "invalid query string") \
- CROW_XX(INVALID_FRAGMENT, "invalid fragment") \
- CROW_XX(LF_EXPECTED, "CROW_LF character expected") \
- CROW_XX(INVALID_HEADER_TOKEN, "invalid character in header") \
- CROW_XX(INVALID_CONTENT_LENGTH, \
- "invalid character in content-length header") \
- CROW_XX(INVALID_CHUNK_SIZE, \
- "invalid character in chunk size header") \
- CROW_XX(INVALID_CONSTANT, "invalid constant string") \
- CROW_XX(INVALID_INTERNAL_STATE, "encountered unexpected internal state")\
- CROW_XX(STRICT, "strict mode assertion failed") \
- CROW_XX(PAUSED, "parser is paused") \
- CROW_XX(UNKNOWN, "an unknown error occurred")
-
-
-/* Define HPE_* values for each errno value above */
-#define CROW_HTTP_ERRNO_GEN(n, s) HPE_##n,
-enum http_errno {
- CROW_HTTP_ERRNO_MAP(CROW_HTTP_ERRNO_GEN)
-};
-#undef CROW_HTTP_ERRNO_GEN
-
-
-/* Get an http_errno value from an http_parser */
-#define CROW_HTTP_PARSER_ERRNO(p) ((enum http_errno) (p)->http_errno)
-
-
-struct http_parser {
- /** PRIVATE **/
- unsigned int type : 2; /* enum http_parser_type */
- unsigned int flags : 6; /* F_* values from 'flags' enum; semi-public */
- unsigned int state : 8; /* enum state from http_parser.c */
- unsigned int header_state : 8; /* enum header_state from http_parser.c */
- unsigned int index : 8; /* index into current matcher */
-
- uint32_t nread; /* # bytes read in various scenarios */
- uint64_t content_length; /* # bytes in body (0 if no Content-Length header) */
-
- /** READ-ONLY **/
- unsigned short http_major;
- unsigned short http_minor;
- unsigned int status_code : 16; /* responses only */
- unsigned int method : 8; /* requests only */
- unsigned int http_errno : 7;
-
- /* 1 = Upgrade header was present and the parser has exited because of that.
- * 0 = No upgrade header present.
- * Should be checked when http_parser_execute() returns in addition to
- * error checking.
- */
- unsigned int upgrade : 1;
-
- /** PUBLIC **/
- void *data; /* A pointer to get hook to the "connection" or "socket" object */
-};
-
-
-struct http_parser_settings {
- http_cb on_message_begin;
- http_data_cb on_url;
- http_data_cb on_status;
- http_data_cb on_header_field;
- http_data_cb on_header_value;
- http_cb on_headers_complete;
- http_data_cb on_body;
- http_cb on_message_complete;
-};
-
-
-enum http_parser_url_fields
- { UF_SCHEMA = 0
- , UF_HOST = 1
- , UF_PORT = 2
- , UF_PATH = 3
- , UF_QUERY = 4
- , UF_FRAGMENT = 5
- , UF_USERINFO = 6
- , UF_MAX = 7
- };
-
-
-/* Result structure for http_parser_parse_url().
- *
- * Callers should index into field_data[] with UF_* values iff field_set
- * has the relevant (1 << UF_*) bit set. As a courtesy to clients (and
- * because we probably have padding left over), we convert any port to
- * a uint16_t.
- */
-struct http_parser_url {
- uint16_t field_set; /* Bitmask of (1 << UF_*) values */
- uint16_t port; /* Converted UF_PORT string */
-
- struct {
- uint16_t off; /* Offset into buffer in which field starts */
- uint16_t len; /* Length of run in buffer */
- } field_data[UF_MAX];
-};
-
-
-/* Returns the library version. Bits 16-23 contain the major version number,
- * bits 8-15 the minor version number and bits 0-7 the patch level.
- * Usage example:
- *
- * unsigned long version = http_parser_version();
- * unsigned major = (version >> 16) & 255;
- * unsigned minor = (version >> 8) & 255;
- * unsigned patch = version & 255;
- * printf("http_parser v%u.%u.%u\n", major, minor, version);
- */
-unsigned long http_parser_version(void);
-
-void http_parser_init(http_parser *parser, enum http_parser_type type);
-
-
-size_t http_parser_execute(http_parser *parser,
- const http_parser_settings *settings,
- const char *data,
- size_t len);
-
-
-/* If http_should_keep_alive() in the on_headers_complete or
- * on_message_complete callback returns 0, then this should be
- * the last message on the connection.
- * If you are the server, respond with the "Connection: close" header.
- * If you are the client, close the connection.
- */
-int http_should_keep_alive(const http_parser *parser);
-
-/* Returns a string version of the HTTP method. */
-const char *http_method_str(enum http_method m);
-
-/* Return a string name of the given error */
-const char *http_errno_name(enum http_errno err);
-
-/* Return a string description of the given error */
-const char *http_errno_description(enum http_errno err);
-
-/* Parse a URL; return nonzero on failure */
-int http_parser_parse_url(const char *buf, size_t buflen,
- int is_connect,
- struct http_parser_url *u);
-
-/* Pause or un-pause the parser; a nonzero value pauses */
-void http_parser_pause(http_parser *parser, int paused);
-
-/* Checks if this is the final chunk of the body. */
-int http_body_is_final(const http_parser *parser);
-
-/*#include "http_parser.h"*/
-/* Based on src/http/ngx_http_parse.c from NGINX copyright Igor Sysoev
- *
- * Additional changes are licensed under the same terms as NGINX and
- * copyright Joyent, Inc. and other Node contributors. All rights reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-#include
-#include
-#include
-#include
-#include
-#include
-
-#ifndef CROW_ULLONG_MAX
-# define CROW_ULLONG_MAX ((uint64_t) -1) /* 2^64-1 */
-#endif
-
-#ifndef CROW_MIN
-# define CROW_MIN(a,b) ((a) < (b) ? (a) : (b))
-#endif
-
-#ifndef CROW_ARRAY_SIZE
-# define CROW_ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
-#endif
-
-#ifndef CROW_BIT_AT
-# define CROW_BIT_AT(a, i) \
- (!!((unsigned int) (a)[(unsigned int) (i) >> 3] & \
- (1 << ((unsigned int) (i) & 7))))
-#endif
-
-#ifndef CROW_ELEM_AT
-# define CROW_ELEM_AT(a, i, v) ((unsigned int) (i) < CROW_ARRAY_SIZE(a) ? (a)[(i)] : (v))
-#endif
-
-#define CROW_SET_ERRNO(e) \
-do { \
- parser->http_errno = (e); \
-} while(0)
-
-
-/* Run the notify callback FOR, returning ER if it fails */
-#define CROW_CALLBACK_NOTIFY_(FOR, ER) \
-do { \
- assert(CROW_HTTP_PARSER_ERRNO(parser) == HPE_OK); \
- \
- if (settings->on_##FOR) { \
- if (0 != settings->on_##FOR(parser)) { \
- CROW_SET_ERRNO(HPE_CB_##FOR); \
- } \
- \
- /* We either errored above or got paused; get out */ \
- if (CROW_HTTP_PARSER_ERRNO(parser) != HPE_OK) { \
- return (ER); \
- } \
- } \
-} while (0)
-
-/* Run the notify callback FOR and consume the current byte */
-#define CROW_CALLBACK_NOTIFY(FOR) CROW_CALLBACK_NOTIFY_(FOR, p - data + 1)
-
-/* Run the notify callback FOR and don't consume the current byte */
-#define CROW_CALLBACK_NOTIFY_NOADVANCE(FOR) CROW_CALLBACK_NOTIFY_(FOR, p - data)
-
-/* Run data callback FOR with LEN bytes, returning ER if it fails */
-#define CROW_CALLBACK_DATA_(FOR, LEN, ER) \
-do { \
- assert(CROW_HTTP_PARSER_ERRNO(parser) == HPE_OK); \
- \
- if (FOR##_mark) { \
- if (settings->on_##FOR) { \
- if (0 != settings->on_##FOR(parser, FOR##_mark, (LEN))) { \
- CROW_SET_ERRNO(HPE_CB_##FOR); \
- } \
- \
- /* We either errored above or got paused; get out */ \
- if (CROW_HTTP_PARSER_ERRNO(parser) != HPE_OK) { \
- return (ER); \
- } \
- } \
- FOR##_mark = NULL; \
- } \
-} while (0)
-
-/* Run the data callback FOR and consume the current byte */
-#define CROW_CALLBACK_DATA(FOR) \
- CROW_CALLBACK_DATA_(FOR, p - FOR##_mark, p - data + 1)
-
-/* Run the data callback FOR and don't consume the current byte */
-#define CROW_CALLBACK_DATA_NOADVANCE(FOR) \
- CROW_CALLBACK_DATA_(FOR, p - FOR##_mark, p - data)
-
-/* Set the mark FOR; non-destructive if mark is already set */
-#define CROW_MARK(FOR) \
-do { \
- if (!FOR##_mark) { \
- FOR##_mark = p; \
- } \
-} while (0)
-
-
-#define CROW_PROXY_CONNECTION "proxy-connection"
-#define CROW_CONNECTION "connection"
-#define CROW_CONTENT_LENGTH "content-length"
-#define CROW_TRANSFER_ENCODING "transfer-encoding"
-#define CROW_UPGRADE "upgrade"
-#define CROW_CHUNKED "chunked"
-#define CROW_KEEP_ALIVE "keep-alive"
-#define CROW_CLOSE "close"
-
-
-
-
-enum state
- { s_dead = 1 /* important that this is > 0 */
-
- , s_start_req_or_res
- , s_res_or_resp_H
- , s_start_res
- , s_res_H
- , s_res_HT
- , s_res_HTT
- , s_res_HTTP
- , s_res_first_http_major
- , s_res_http_major
- , s_res_first_http_minor
- , s_res_http_minor
- , s_res_first_status_code
- , s_res_status_code
- , s_res_status_start
- , s_res_status
- , s_res_line_almost_done
-
- , s_start_req
-
- , s_req_method
- , s_req_spaces_before_url
- , s_req_schema
- , s_req_schema_slash
- , s_req_schema_slash_slash
- , s_req_server_start
- , s_req_server
- , s_req_server_with_at
- , s_req_path
- , s_req_query_string_start
- , s_req_query_string
- , s_req_fragment_start
- , s_req_fragment
- , s_req_http_start
- , s_req_http_H
- , s_req_http_HT
- , s_req_http_HTT
- , s_req_http_HTTP
- , s_req_first_http_major
- , s_req_http_major
- , s_req_first_http_minor
- , s_req_http_minor
- , s_req_line_almost_done
-
- , s_header_field_start
- , s_header_field
- , s_header_value_discard_ws
- , s_header_value_discard_ws_almost_done
- , s_header_value_discard_lws
- , s_header_value_start
- , s_header_value
- , s_header_value_lws
-
- , s_header_almost_done
-
- , s_chunk_size_start
- , s_chunk_size
- , s_chunk_parameters
- , s_chunk_size_almost_done
-
- , s_headers_almost_done
- , s_headers_done
-
- /* Important: 's_headers_done' must be the last 'header' state. All
- * states beyond this must be 'body' states. It is used for overflow
- * checking. See the CROW_PARSING_HEADER() macro.
- */
-
- , s_chunk_data
- , s_chunk_data_almost_done
- , s_chunk_data_done
-
- , s_body_identity
- , s_body_identity_eof
-
- , s_message_done
- };
-
-
-#define CROW_PARSING_HEADER(state) (state <= s_headers_done)
-
-
-enum header_states
- { h_general = 0
- , h_C
- , h_CO
- , h_CON
-
- , h_matching_connection
- , h_matching_proxy_connection
- , h_matching_content_length
- , h_matching_transfer_encoding
- , h_matching_upgrade
-
- , h_connection
- , h_content_length
- , h_transfer_encoding
- , h_upgrade
-
- , h_matching_transfer_encoding_chunked
- , h_matching_connection_keep_alive
- , h_matching_connection_close
-
- , h_transfer_encoding_chunked
- , h_connection_keep_alive
- , h_connection_close
- };
-
-enum http_host_state
- {
- s_http_host_dead = 1
- , s_http_userinfo_start
- , s_http_userinfo
- , s_http_host_start
- , s_http_host_v6_start
- , s_http_host
- , s_http_host_v6
- , s_http_host_v6_end
- , s_http_host_port_start
- , s_http_host_port
-};
-
-/* Macros for character classes; depends on strict-mode */
-#define CROW_CR '\r'
-#define CROW_LF '\n'
-#define CROW_LOWER(c) (unsigned char)(c | 0x20)
-#define CROW_IS_ALPHA(c) (CROW_LOWER(c) >= 'a' && CROW_LOWER(c) <= 'z')
-#define CROW_IS_NUM(c) ((c) >= '0' && (c) <= '9')
-#define CROW_IS_ALPHANUM(c) (CROW_IS_ALPHA(c) || CROW_IS_NUM(c))
-#define CROW_IS_HEX(c) (CROW_IS_NUM(c) || (CROW_LOWER(c) >= 'a' && CROW_LOWER(c) <= 'f'))
-#define CROW_IS_MARK(c) ((c) == '-' || (c) == '_' || (c) == '.' || \
- (c) == '!' || (c) == '~' || (c) == '*' || (c) == '\'' || (c) == '(' || \
- (c) == ')')
-#define CROW_IS_USERINFO_CHAR(c) (CROW_IS_ALPHANUM(c) || CROW_IS_MARK(c) || (c) == '%' || \
- (c) == ';' || (c) == ':' || (c) == '&' || (c) == '=' || (c) == '+' || \
- (c) == '$' || (c) == ',')
-
-#if CROW_HTTP_PARSER_STRICT
-#define CROW_TOKEN(c) (tokens[(unsigned char)c])
-#define CROW_IS_URL_CHAR(c) (CROW_BIT_AT(normal_url_char, (unsigned char)c))
-#define CROW_IS_HOST_CHAR(c) (CROW_IS_ALPHANUM(c) || (c) == '.' || (c) == '-')
-#else
-#define CROW_TOKEN(c) ((c == ' ') ? ' ' : tokens[(unsigned char)c])
-#define CROW_IS_URL_CHAR(c) \
- (CROW_BIT_AT(normal_url_char, (unsigned char)c) || ((c) & 0x80))
-#define CROW_IS_HOST_CHAR(c) \
- (CROW_IS_ALPHANUM(c) || (c) == '.' || (c) == '-' || (c) == '_')
-#endif
-
-
-#define CROW_start_state (parser->type == HTTP_REQUEST ? s_start_req : s_start_res)
-
-
-#if CROW_HTTP_PARSER_STRICT
-# define CROW_STRICT_CHECK(cond) \
-do { \
- if (cond) { \
- CROW_SET_ERRNO(HPE_STRICT); \
- goto error; \
- } \
-} while (0)
-# define CROW_NEW_MESSAGE() (http_should_keep_alive(parser) ? CROW_start_state : s_dead)
-#else
-# define CROW_STRICT_CHECK(cond)
-# define CROW_NEW_MESSAGE() CROW_start_state
-#endif
-
-
-
-int http_message_needs_eof(const http_parser *parser);
-
-/* Our URL parser.
- *
- * This is designed to be shared by http_parser_execute() for URL validation,
- * hence it has a state transition + byte-for-byte interface. In addition, it
- * is meant to be embedded in http_parser_parse_url(), which does the dirty
- * work of turning state transitions URL components for its API.
- *
- * This function should only be invoked with non-space characters. It is
- * assumed that the caller cares about (and can detect) the transition between
- * URL and non-URL states by looking for these.
- */
-inline enum state
-parse_url_char(enum state s, const char ch)
-{
-#if CROW_HTTP_PARSER_STRICT
-# define CROW_T(v) 0
-#else
-# define CROW_T(v) v
-#endif
-
-
-static const uint8_t normal_url_char[32] = {
-/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */
- 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0,
-/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */
- 0 | CROW_T(2) | 0 | 0 | CROW_T(16) | 0 | 0 | 0,
-/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */
- 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0,
-/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */
- 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0,
-/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */
- 0 | 2 | 4 | 0 | 16 | 32 | 64 | 128,
-/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */
- 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
-/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */
- 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
-/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */
- 1 | 2 | 4 | 8 | 16 | 32 | 64 | 0,
-/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */
- 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
-/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */
- 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
-/* 80 P 81 Q 82 R 83 S 84 CROW_T 85 U 86 V 87 W */
- 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
-/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */
- 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
-/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */
- 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
-/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */
- 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
-/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */
- 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128,
-/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */
- 1 | 2 | 4 | 8 | 16 | 32 | 64 | 0, };
-
-#undef CROW_T
-
- if (ch == ' ' || ch == '\r' || ch == '\n') {
- return s_dead;
- }
-
-#if CROW_HTTP_PARSER_STRICT
- if (ch == '\t' || ch == '\f') {
- return s_dead;
- }
-#endif
-
- switch (s) {
- case s_req_spaces_before_url:
- /* Proxied requests are followed by scheme of an absolute URI (alpha).
- * All methods except CONNECT are followed by '/' or '*'.
- */
-
- if (ch == '/' || ch == '*') {
- return s_req_path;
- }
-
- if (CROW_IS_ALPHA(ch)) {
- return s_req_schema;
- }
-
- break;
-
- case s_req_schema:
- if (CROW_IS_ALPHA(ch)) {
- return s;
- }
-
- if (ch == ':') {
- return s_req_schema_slash;
- }
-
- break;
-
- case s_req_schema_slash:
- if (ch == '/') {
- return s_req_schema_slash_slash;
- }
-
- break;
-
- case s_req_schema_slash_slash:
- if (ch == '/') {
- return s_req_server_start;
- }
-
- break;
-
- case s_req_server_with_at:
- if (ch == '@') {
- return s_dead;
- }
-
- /* FALLTHROUGH */
- case s_req_server_start:
- case s_req_server:
- if (ch == '/') {
- return s_req_path;
- }
-
- if (ch == '?') {
- return s_req_query_string_start;
- }
-
- if (ch == '@') {
- return s_req_server_with_at;
- }
-
- if (CROW_IS_USERINFO_CHAR(ch) || ch == '[' || ch == ']') {
- return s_req_server;
- }
-
- break;
-
- case s_req_path:
- if (CROW_IS_URL_CHAR(ch)) {
- return s;
- }
-
- switch (ch) {
- case '?':
- return s_req_query_string_start;
-
- case '#':
- return s_req_fragment_start;
- }
-
- break;
-
- case s_req_query_string_start:
- case s_req_query_string:
- if (CROW_IS_URL_CHAR(ch)) {
- return s_req_query_string;
- }
-
- switch (ch) {
- case '?':
- /* allow extra '?' in query string */
- return s_req_query_string;
-
- case '#':
- return s_req_fragment_start;
- }
-
- break;
-
- case s_req_fragment_start:
- if (CROW_IS_URL_CHAR(ch)) {
- return s_req_fragment;
- }
-
- switch (ch) {
- case '?':
- return s_req_fragment;
-
- case '#':
- return s;
- }
-
- break;
-
- case s_req_fragment:
- if (CROW_IS_URL_CHAR(ch)) {
- return s;
- }
-
- switch (ch) {
- case '?':
- case '#':
- return s;
- }
-
- break;
-
- default:
- break;
- }
-
- /* We should never fall out of the switch above unless there's an error */
- return s_dead;
-}
-
-inline size_t http_parser_execute (http_parser *parser,
- const http_parser_settings *settings,
- const char *data,
- size_t len)
-{
-static const char *method_strings[] =
- {
-#define CROW_XX(num, name, string) #string,
- CROW_HTTP_METHOD_MAP(CROW_XX)
-#undef CROW_XX
- };
-
-/* Tokens as defined by rfc 2616. Also lowercases them.
- * token = 1*
- * separators = "(" | ")" | "<" | ">" | "@"
- * | "," | ";" | ":" | "\" | <">
- * | "/" | "[" | "]" | "?" | "="
- * | "{" | "}" | SP | HT
- */
-static const char tokens[256] = {
-/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */
- 0, 0, 0, 0, 0, 0, 0, 0,
-/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */
- 0, 0, 0, 0, 0, 0, 0, 0,
-/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */
- 0, 0, 0, 0, 0, 0, 0, 0,
-/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */
- 0, 0, 0, 0, 0, 0, 0, 0,
-/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */
- 0, '!', 0, '#', '$', '%', '&', '\'',
-/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */
- 0, 0, '*', '+', 0, '-', '.', 0,
-/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */
- '0', '1', '2', '3', '4', '5', '6', '7',
-/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */
- '8', '9', 0, 0, 0, 0, 0, 0,
-/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */
- 0, 'a', 'b', 'c', 'd', 'e', 'f', 'g',
-/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */
- 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
-/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */
- 'p', 'q', 'r', 's', 't', 'u', 'v', 'w',
-/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */
- 'x', 'y', 'z', 0, 0, 0, '^', '_',
-/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */
- '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g',
-/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */
- 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
-/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */
- 'p', 'q', 'r', 's', 't', 'u', 'v', 'w',
-/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */
- 'x', 'y', 'z', 0, '|', 0, '~', 0 };
-
-
-static const int8_t unhex[256] =
- {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
- ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
- ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
- , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1
- ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
- ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
- ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
- ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
- };
-
-
-
- char c, ch;
- int8_t unhex_val;
- const char *p = data;
- const char *header_field_mark = 0;
- const char *header_value_mark = 0;
- const char *url_mark = 0;
- const char *body_mark = 0;
- const char *status_mark = 0;
-
- /* We're in an error state. Don't bother doing anything. */
- if (CROW_HTTP_PARSER_ERRNO(parser) != HPE_OK) {
- return 0;
- }
-
- if (len == 0) {
- switch (parser->state) {
- case s_body_identity_eof:
- /* Use of CROW_CALLBACK_NOTIFY() here would erroneously return 1 byte read if
- * we got paused.
- */
- CROW_CALLBACK_NOTIFY_NOADVANCE(message_complete);
- return 0;
-
- case s_dead:
- case s_start_req_or_res:
- case s_start_res:
- case s_start_req:
- return 0;
-
- default:
- CROW_SET_ERRNO(HPE_INVALID_EOF_STATE);
- return 1;
- }
- }
-
-
- if (parser->state == s_header_field)
- header_field_mark = data;
- if (parser->state == s_header_value)
- header_value_mark = data;
- switch (parser->state) {
- case s_req_path:
- case s_req_schema:
- case s_req_schema_slash:
- case s_req_schema_slash_slash:
- case s_req_server_start:
- case s_req_server:
- case s_req_server_with_at:
- case s_req_query_string_start:
- case s_req_query_string:
- case s_req_fragment_start:
- case s_req_fragment:
- url_mark = data;
- break;
- case s_res_status:
- status_mark = data;
- break;
- }
-
- for (p=data; p != data + len; p++) {
- ch = *p;
-
- if (CROW_PARSING_HEADER(parser->state)) {
- ++parser->nread;
- /* Don't allow the total size of the HTTP headers (including the status
- * line) to exceed CROW_HTTP_MAX_HEADER_SIZE. This check is here to protect
- * embedders against denial-of-service attacks where the attacker feeds
- * us a never-ending header that the embedder keeps buffering.
- *
- * This check is arguably the responsibility of embedders but we're doing
- * it on the embedder's behalf because most won't bother and this way we
- * make the web a little safer. CROW_HTTP_MAX_HEADER_SIZE is still far bigger
- * than any reasonable request or response so this should never affect
- * day-to-day operation.
- */
- if (parser->nread > (CROW_HTTP_MAX_HEADER_SIZE)) {
- CROW_SET_ERRNO(HPE_HEADER_OVERFLOW);
- goto error;
- }
- }
-
- reexecute_byte:
- switch (parser->state) {
-
- case s_dead:
- /* this state is used after a 'Connection: close' message
- * the parser will error out if it reads another message
- */
- if (ch == CROW_CR || ch == CROW_LF)
- break;
-
- CROW_SET_ERRNO(HPE_CLOSED_CONNECTION);
- goto error;
-
- case s_start_req_or_res:
- {
- if (ch == CROW_CR || ch == CROW_LF)
- break;
- parser->flags = 0;
- parser->content_length = CROW_ULLONG_MAX;
-
- if (ch == 'H') {
- parser->state = s_res_or_resp_H;
-
- CROW_CALLBACK_NOTIFY(message_begin);
- } else {
- parser->type = HTTP_REQUEST;
- parser->state = s_start_req;
- goto reexecute_byte;
- }
-
- break;
- }
-
- case s_res_or_resp_H:
- if (ch == 'T') {
- parser->type = HTTP_RESPONSE;
- parser->state = s_res_HT;
- } else {
- if (ch != 'E') {
- CROW_SET_ERRNO(HPE_INVALID_CONSTANT);
- goto error;
- }
-
- parser->type = HTTP_REQUEST;
- parser->method = HTTP_HEAD;
- parser->index = 2;
- parser->state = s_req_method;
- }
- break;
-
- case s_start_res:
- {
- parser->flags = 0;
- parser->content_length = CROW_ULLONG_MAX;
-
- switch (ch) {
- case 'H':
- parser->state = s_res_H;
- break;
-
- case CROW_CR:
- case CROW_LF:
- break;
-
- default:
- CROW_SET_ERRNO(HPE_INVALID_CONSTANT);
- goto error;
- }
-
- CROW_CALLBACK_NOTIFY(message_begin);
- break;
- }
-
- case s_res_H:
- CROW_STRICT_CHECK(ch != 'T');
- parser->state = s_res_HT;
- break;
-
- case s_res_HT:
- CROW_STRICT_CHECK(ch != 'T');
- parser->state = s_res_HTT;
- break;
-
- case s_res_HTT:
- CROW_STRICT_CHECK(ch != 'P');
- parser->state = s_res_HTTP;
- break;
-
- case s_res_HTTP:
- CROW_STRICT_CHECK(ch != '/');
- parser->state = s_res_first_http_major;
- break;
-
- case s_res_first_http_major:
- if (ch < '0' || ch > '9') {
- CROW_SET_ERRNO(HPE_INVALID_VERSION);
- goto error;
- }
-
- parser->http_major = ch - '0';
- parser->state = s_res_http_major;
- break;
-
- /* major HTTP version or dot */
- case s_res_http_major:
- {
- if (ch == '.') {
- parser->state = s_res_first_http_minor;
- break;
- }
-
- if (!CROW_IS_NUM(ch)) {
- CROW_SET_ERRNO(HPE_INVALID_VERSION);
- goto error;
- }
-
- parser->http_major *= 10;
- parser->http_major += ch - '0';
-
- if (parser->http_major > 999) {
- CROW_SET_ERRNO(HPE_INVALID_VERSION);
- goto error;
- }
-
- break;
- }
-
- /* first digit of minor HTTP version */
- case s_res_first_http_minor:
- if (!CROW_IS_NUM(ch)) {
- CROW_SET_ERRNO(HPE_INVALID_VERSION);
- goto error;
- }
-
- parser->http_minor = ch - '0';
- parser->state = s_res_http_minor;
- break;
-
- /* minor HTTP version or end of request line */
- case s_res_http_minor:
- {
- if (ch == ' ') {
- parser->state = s_res_first_status_code;
- break;
- }
-
- if (!CROW_IS_NUM(ch)) {
- CROW_SET_ERRNO(HPE_INVALID_VERSION);
- goto error;
- }
-
- parser->http_minor *= 10;
- parser->http_minor += ch - '0';
-
- if (parser->http_minor > 999) {
- CROW_SET_ERRNO(HPE_INVALID_VERSION);
- goto error;
- }
-
- break;
- }
-
- case s_res_first_status_code:
- {
- if (!CROW_IS_NUM(ch)) {
- if (ch == ' ') {
- break;
- }
-
- CROW_SET_ERRNO(HPE_INVALID_STATUS);
- goto error;
- }
- parser->status_code = ch - '0';
- parser->state = s_res_status_code;
- break;
- }
-
- case s_res_status_code:
- {
- if (!CROW_IS_NUM(ch)) {
- switch (ch) {
- case ' ':
- parser->state = s_res_status_start;
- break;
- case CROW_CR:
- parser->state = s_res_line_almost_done;
- break;
- case CROW_LF:
- parser->state = s_header_field_start;
- break;
- default:
- CROW_SET_ERRNO(HPE_INVALID_STATUS);
- goto error;
- }
- break;
- }
-
- parser->status_code *= 10;
- parser->status_code += ch - '0';
-
- if (parser->status_code > 999) {
- CROW_SET_ERRNO(HPE_INVALID_STATUS);
- goto error;
- }
-
- break;
- }
-
- case s_res_status_start:
- {
- if (ch == CROW_CR) {
- parser->state = s_res_line_almost_done;
- break;
- }
-
- if (ch == CROW_LF) {
- parser->state = s_header_field_start;
- break;
- }
-
- CROW_MARK(status);
- parser->state = s_res_status;
- parser->index = 0;
- break;
- }
-
- case s_res_status:
- if (ch == CROW_CR) {
- parser->state = s_res_line_almost_done;
- CROW_CALLBACK_DATA(status);
- break;
- }
-
- if (ch == CROW_LF) {
- parser->state = s_header_field_start;
- CROW_CALLBACK_DATA(status);
- break;
- }
-
- break;
-
- case s_res_line_almost_done:
- CROW_STRICT_CHECK(ch != CROW_LF);
- parser->state = s_header_field_start;
- break;
-
- case s_start_req:
- {
- if (ch == CROW_CR || ch == CROW_LF)
- break;
- parser->flags = 0;
- parser->content_length = CROW_ULLONG_MAX;
-
- if (!CROW_IS_ALPHA(ch)) {
- CROW_SET_ERRNO(HPE_INVALID_METHOD);
- goto error;
- }
-
- parser->method = (enum http_method) 0;
- parser->index = 1;
- switch (ch) {
- case 'C': parser->method = HTTP_CONNECT; /* or COPY, CHECKOUT */ break;
- case 'D': parser->method = HTTP_DELETE; break;
- case 'G': parser->method = HTTP_GET; break;
- case 'H': parser->method = HTTP_HEAD; break;
- case 'L': parser->method = HTTP_LOCK; break;
- case 'M': parser->method = HTTP_MKCOL; /* or MOVE, MKACTIVITY, MERGE, M-SEARCH, MKCALENDAR */ break;
- case 'N': parser->method = HTTP_NOTIFY; break;
- case 'O': parser->method = HTTP_OPTIONS; break;
- case 'P': parser->method = HTTP_POST;
- /* or PROPFIND|PROPPATCH|PUT|PATCH|PURGE */
- break;
- case 'R': parser->method = HTTP_REPORT; break;
- case 'S': parser->method = HTTP_SUBSCRIBE; /* or SEARCH */ break;
- case 'T': parser->method = HTTP_TRACE; break;
- case 'U': parser->method = HTTP_UNLOCK; /* or UNSUBSCRIBE */ break;
- default:
- CROW_SET_ERRNO(HPE_INVALID_METHOD);
- goto error;
- }
- parser->state = s_req_method;
-
- CROW_CALLBACK_NOTIFY(message_begin);
-
- break;
- }
-
- case s_req_method:
- {
- const char *matcher;
- if (ch == '\0') {
- CROW_SET_ERRNO(HPE_INVALID_METHOD);
- goto error;
- }
-
- matcher = method_strings[parser->method];
- if (ch == ' ' && matcher[parser->index] == '\0') {
- parser->state = s_req_spaces_before_url;
- } else if (ch == matcher[parser->index]) {
- ; /* nada */
- } else if (parser->method == HTTP_CONNECT) {
- if (parser->index == 1 && ch == 'H') {
- parser->method = HTTP_CHECKOUT;
- } else if (parser->index == 2 && ch == 'P') {
- parser->method = HTTP_COPY;
- } else {
- CROW_SET_ERRNO(HPE_INVALID_METHOD);
- goto error;
- }
- } else if (parser->method == HTTP_MKCOL) {
- if (parser->index == 1 && ch == 'O') {
- parser->method = HTTP_MOVE;
- } else if (parser->index == 1 && ch == 'E') {
- parser->method = HTTP_MERGE;
- } else if (parser->index == 1 && ch == '-') {
- parser->method = HTTP_MSEARCH;
- } else if (parser->index == 2 && ch == 'A') {
- parser->method = HTTP_MKACTIVITY;
- } else if (parser->index == 3 && ch == 'A') {
- parser->method = HTTP_MKCALENDAR;
- } else {
- CROW_SET_ERRNO(HPE_INVALID_METHOD);
- goto error;
- }
- } else if (parser->method == HTTP_SUBSCRIBE) {
- if (parser->index == 1 && ch == 'E') {
- parser->method = HTTP_SEARCH;
- } else {
- CROW_SET_ERRNO(HPE_INVALID_METHOD);
- goto error;
- }
- } else if (parser->index == 1 && parser->method == HTTP_POST) {
- if (ch == 'R') {
- parser->method = HTTP_PROPFIND; /* or HTTP_PROPPATCH */
- } else if (ch == 'U') {
- parser->method = HTTP_PUT; /* or HTTP_PURGE */
- } else if (ch == 'A') {
- parser->method = HTTP_PATCH;
- } else {
- CROW_SET_ERRNO(HPE_INVALID_METHOD);
- goto error;
- }
- } else if (parser->index == 2) {
- if (parser->method == HTTP_PUT) {
- if (ch == 'R') {
- parser->method = HTTP_PURGE;
- } else {
- CROW_SET_ERRNO(HPE_INVALID_METHOD);
- goto error;
- }
- } else if (parser->method == HTTP_UNLOCK) {
- if (ch == 'S') {
- parser->method = HTTP_UNSUBSCRIBE;
- } else {
- CROW_SET_ERRNO(HPE_INVALID_METHOD);
- goto error;
- }
- } else {
- CROW_SET_ERRNO(HPE_INVALID_METHOD);
- goto error;
- }
- } else if (parser->index == 4 && parser->method == HTTP_PROPFIND && ch == 'P') {
- parser->method = HTTP_PROPPATCH;
- } else {
- CROW_SET_ERRNO(HPE_INVALID_METHOD);
- goto error;
- }
-
- ++parser->index;
- break;
- }
-
- case s_req_spaces_before_url:
- {
- if (ch == ' ') break;
-
- CROW_MARK(url);
- if (parser->method == HTTP_CONNECT) {
- parser->state = s_req_server_start;
- }
-
- parser->state = parse_url_char((enum state)parser->state, ch);
- if (parser->state == s_dead) {
- CROW_SET_ERRNO(HPE_INVALID_URL);
- goto error;
- }
-
- break;
- }
-
- case s_req_schema:
- case s_req_schema_slash:
- case s_req_schema_slash_slash:
- case s_req_server_start:
- {
- switch (ch) {
- /* No whitespace allowed here */
- case ' ':
- case CROW_CR:
- case CROW_LF:
- CROW_SET_ERRNO(HPE_INVALID_URL);
- goto error;
- default:
- parser->state = parse_url_char((enum state)parser->state, ch);
- if (parser->state == s_dead) {
- CROW_SET_ERRNO(HPE_INVALID_URL);
- goto error;
- }
- }
-
- break;
- }
-
- case s_req_server:
- case s_req_server_with_at:
- case s_req_path:
- case s_req_query_string_start:
- case s_req_query_string:
- case s_req_fragment_start:
- case s_req_fragment:
- {
- switch (ch) {
- case ' ':
- parser->state = s_req_http_start;
- CROW_CALLBACK_DATA(url);
- break;
- case CROW_CR:
- case CROW_LF:
- parser->http_major = 0;
- parser->http_minor = 9;
- parser->state = (ch == CROW_CR) ?
- s_req_line_almost_done :
- s_header_field_start;
- CROW_CALLBACK_DATA(url);
- break;
- default:
- parser->state = parse_url_char((enum state)parser->state, ch);
- if (parser->state == s_dead) {
- CROW_SET_ERRNO(HPE_INVALID_URL);
- goto error;
- }
- }
- break;
- }
-
- case s_req_http_start:
- switch (ch) {
- case 'H':
- parser->state = s_req_http_H;
- break;
- case ' ':
- break;
- default:
- CROW_SET_ERRNO(HPE_INVALID_CONSTANT);
- goto error;
- }
- break;
-
- case s_req_http_H:
- CROW_STRICT_CHECK(ch != 'T');
- parser->state = s_req_http_HT;
- break;
-
- case s_req_http_HT:
- CROW_STRICT_CHECK(ch != 'T');
- parser->state = s_req_http_HTT;
- break;
-
- case s_req_http_HTT:
- CROW_STRICT_CHECK(ch != 'P');
- parser->state = s_req_http_HTTP;
- break;
-
- case s_req_http_HTTP:
- CROW_STRICT_CHECK(ch != '/');
- parser->state = s_req_first_http_major;
- break;
-
- /* first digit of major HTTP version */
- case s_req_first_http_major:
- if (ch < '1' || ch > '9') {
- CROW_SET_ERRNO(HPE_INVALID_VERSION);
- goto error;
- }
-
- parser->http_major = ch - '0';
- parser->state = s_req_http_major;
- break;
-
- /* major HTTP version or dot */
- case s_req_http_major:
- {
- if (ch == '.') {
- parser->state = s_req_first_http_minor;
- break;
- }
-
- if (!CROW_IS_NUM(ch)) {
- CROW_SET_ERRNO(HPE_INVALID_VERSION);
- goto error;
- }
-
- parser->http_major *= 10;
- parser->http_major += ch - '0';
-
- if (parser->http_major > 999) {
- CROW_SET_ERRNO(HPE_INVALID_VERSION);
- goto error;
- }
-
- break;
- }
-
- /* first digit of minor HTTP version */
- case s_req_first_http_minor:
- if (!CROW_IS_NUM(ch)) {
- CROW_SET_ERRNO(HPE_INVALID_VERSION);
- goto error;
- }
-
- parser->http_minor = ch - '0';
- parser->state = s_req_http_minor;
- break;
-
- /* minor HTTP version or end of request line */
- case s_req_http_minor:
- {
- if (ch == CROW_CR) {
- parser->state = s_req_line_almost_done;
- break;
- }
-
- if (ch == CROW_LF) {
- parser->state = s_header_field_start;
- break;
- }
-
- /* XXX allow spaces after digit? */
-
- if (!CROW_IS_NUM(ch)) {
- CROW_SET_ERRNO(HPE_INVALID_VERSION);
- goto error;
- }
-
- parser->http_minor *= 10;
- parser->http_minor += ch - '0';
-
- if (parser->http_minor > 999) {
- CROW_SET_ERRNO(HPE_INVALID_VERSION);
- goto error;
- }
-
- break;
- }
-
- /* end of request line */
- case s_req_line_almost_done:
- {
- if (ch != CROW_LF) {
- CROW_SET_ERRNO(HPE_LF_EXPECTED);
- goto error;
- }
-
- parser->state = s_header_field_start;
- break;
- }
-
- case s_header_field_start:
- {
- if (ch == CROW_CR) {
- parser->state = s_headers_almost_done;
- break;
- }
-
- if (ch == CROW_LF) {
- /* they might be just sending \n instead of \r\n so this would be
- * the second \n to denote the end of headers*/
- parser->state = s_headers_almost_done;
- goto reexecute_byte;
- }
-
- c = CROW_TOKEN(ch);
-
- if (!c) {
- CROW_SET_ERRNO(HPE_INVALID_HEADER_TOKEN);
- goto error;
- }
-
- CROW_MARK(header_field);
-
- parser->index = 0;
- parser->state = s_header_field;
-
- switch (c) {
- case 'c':
- parser->header_state = h_C;
- break;
-
- case 'p':
- parser->header_state = h_matching_proxy_connection;
- break;
-
- case 't':
- parser->header_state = h_matching_transfer_encoding;
- break;
-
- case 'u':
- parser->header_state = h_matching_upgrade;
- break;
-
- default:
- parser->header_state = h_general;
- break;
- }
- break;
- }
-
- case s_header_field:
- {
- c = CROW_TOKEN(ch);
-
- if (c) {
- switch (parser->header_state) {
- case h_general:
- break;
-
- case h_C:
- parser->index++;
- parser->header_state = (c == 'o' ? h_CO : h_general);
- break;
-
- case h_CO:
- parser->index++;
- parser->header_state = (c == 'n' ? h_CON : h_general);
- break;
-
- case h_CON:
- parser->index++;
- switch (c) {
- case 'n':
- parser->header_state = h_matching_connection;
- break;
- case 't':
- parser->header_state = h_matching_content_length;
- break;
- default:
- parser->header_state = h_general;
- break;
- }
- break;
-
- /* connection */
-
- case h_matching_connection:
- parser->index++;
- if (parser->index > sizeof(CROW_CONNECTION)-1
- || c != CROW_CONNECTION[parser->index]) {
- parser->header_state = h_general;
- } else if (parser->index == sizeof(CROW_CONNECTION)-2) {
- parser->header_state = h_connection;
- }
- break;
-
- /* proxy-connection */
-
- case h_matching_proxy_connection:
- parser->index++;
- if (parser->index > sizeof(CROW_PROXY_CONNECTION)-1
- || c != CROW_PROXY_CONNECTION[parser->index]) {
- parser->header_state = h_general;
- } else if (parser->index == sizeof(CROW_PROXY_CONNECTION)-2) {
- parser->header_state = h_connection;
- }
- break;
-
- /* content-length */
-
- case h_matching_content_length:
- parser->index++;
- if (parser->index > sizeof(CROW_CONTENT_LENGTH)-1
- || c != CROW_CONTENT_LENGTH[parser->index]) {
- parser->header_state = h_general;
- } else if (parser->index == sizeof(CROW_CONTENT_LENGTH)-2) {
- parser->header_state = h_content_length;
- }
- break;
-
- /* transfer-encoding */
-
- case h_matching_transfer_encoding:
- parser->index++;
- if (parser->index > sizeof(CROW_TRANSFER_ENCODING)-1
- || c != CROW_TRANSFER_ENCODING[parser->index]) {
- parser->header_state = h_general;
- } else if (parser->index == sizeof(CROW_TRANSFER_ENCODING)-2) {
- parser->header_state = h_transfer_encoding;
- }
- break;
-
- /* upgrade */
-
- case h_matching_upgrade:
- parser->index++;
- if (parser->index > sizeof(CROW_UPGRADE)-1
- || c != CROW_UPGRADE[parser->index]) {
- parser->header_state = h_general;
- } else if (parser->index == sizeof(CROW_UPGRADE)-2) {
- parser->header_state = h_upgrade;
- }
- break;
-
- case h_connection:
- case h_content_length:
- case h_transfer_encoding:
- case h_upgrade:
- if (ch != ' ') parser->header_state = h_general;
- break;
-
- default:
- assert(0 && "Unknown header_state");
- break;
- }
- break;
- }
-
- if (ch == ':') {
- parser->state = s_header_value_discard_ws;
- CROW_CALLBACK_DATA(header_field);
- break;
- }
-
- if (ch == CROW_CR) {
- parser->state = s_header_almost_done;
- CROW_CALLBACK_DATA(header_field);
- break;
- }
-
- if (ch == CROW_LF) {
- parser->state = s_header_field_start;
- CROW_CALLBACK_DATA(header_field);
- break;
- }
-
- CROW_SET_ERRNO(HPE_INVALID_HEADER_TOKEN);
- goto error;
- }
-
- case s_header_value_discard_ws:
- if (ch == ' ' || ch == '\t') break;
-
- if (ch == CROW_CR) {
- parser->state = s_header_value_discard_ws_almost_done;
- break;
- }
-
- if (ch == CROW_LF) {
- parser->state = s_header_value_discard_lws;
- break;
- }
-
- /* FALLTHROUGH */
-
- case s_header_value_start:
- {
- CROW_MARK(header_value);
-
- parser->state = s_header_value;
- parser->index = 0;
-
- c = CROW_LOWER(ch);
-
- switch (parser->header_state) {
- case h_upgrade:
- parser->flags |= F_UPGRADE;
- parser->header_state = h_general;
- break;
-
- case h_transfer_encoding:
- /* looking for 'Transfer-Encoding: chunked' */
- if ('c' == c) {
- parser->header_state = h_matching_transfer_encoding_chunked;
- } else {
- parser->header_state = h_general;
- }
- break;
-
- case h_content_length:
- if (!CROW_IS_NUM(ch)) {
- CROW_SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
- goto error;
- }
-
- parser->content_length = ch - '0';
- break;
-
- case h_connection:
- /* looking for 'Connection: keep-alive' */
- if (c == 'k') {
- parser->header_state = h_matching_connection_keep_alive;
- /* looking for 'Connection: close' */
- } else if (c == 'c') {
- parser->header_state = h_matching_connection_close;
- } else {
- parser->header_state = h_general;
- }
- break;
-
- default:
- parser->header_state = h_general;
- break;
- }
- break;
- }
-
- case s_header_value:
- {
-
- if (ch == CROW_CR) {
- parser->state = s_header_almost_done;
- CROW_CALLBACK_DATA(header_value);
- break;
- }
-
- if (ch == CROW_LF) {
- parser->state = s_header_almost_done;
- CROW_CALLBACK_DATA_NOADVANCE(header_value);
- goto reexecute_byte;
- }
-
- c = CROW_LOWER(ch);
-
- switch (parser->header_state) {
- case h_general:
- break;
-
- case h_connection:
- case h_transfer_encoding:
- assert(0 && "Shouldn't get here.");
- break;
-
- case h_content_length:
- {
- uint64_t t;
-
- if (ch == ' ') break;
-
- if (!CROW_IS_NUM(ch)) {
- CROW_SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
- goto error;
- }
-
- t = parser->content_length;
- t *= 10;
- t += ch - '0';
-
- /* Overflow? Test against a conservative limit for simplicity. */
- if ((CROW_ULLONG_MAX - 10) / 10 < parser->content_length) {
- CROW_SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
- goto error;
- }
-
- parser->content_length = t;
- break;
- }
-
- /* Transfer-Encoding: chunked */
- case h_matching_transfer_encoding_chunked:
- parser->index++;
- if (parser->index > sizeof(CROW_CHUNKED)-1
- || c != CROW_CHUNKED[parser->index]) {
- parser->header_state = h_general;
- } else if (parser->index == sizeof(CROW_CHUNKED)-2) {
- parser->header_state = h_transfer_encoding_chunked;
- }
- break;
-
- /* looking for 'Connection: keep-alive' */
- case h_matching_connection_keep_alive:
- parser->index++;
- if (parser->index > sizeof(CROW_KEEP_ALIVE)-1
- || c != CROW_KEEP_ALIVE[parser->index]) {
- parser->header_state = h_general;
- } else if (parser->index == sizeof(CROW_KEEP_ALIVE)-2) {
- parser->header_state = h_connection_keep_alive;
- }
- break;
-
- /* looking for 'Connection: close' */
- case h_matching_connection_close:
- parser->index++;
- if (parser->index > sizeof(CROW_CLOSE)-1 || c != CROW_CLOSE[parser->index]) {
- parser->header_state = h_general;
- } else if (parser->index == sizeof(CROW_CLOSE)-2) {
- parser->header_state = h_connection_close;
- }
- break;
-
- case h_transfer_encoding_chunked:
- case h_connection_keep_alive:
- case h_connection_close:
- if (ch != ' ') parser->header_state = h_general;
- break;
-
- default:
- parser->state = s_header_value;
- parser->header_state = h_general;
- break;
- }
- break;
- }
-
- case s_header_almost_done:
- {
- CROW_STRICT_CHECK(ch != CROW_LF);
-
- parser->state = s_header_value_lws;
- break;
- }
-
- case s_header_value_lws:
- {
- if (ch == ' ' || ch == '\t') {
- parser->state = s_header_value_start;
- goto reexecute_byte;
- }
-
- /* finished the header */
- switch (parser->header_state) {
- case h_connection_keep_alive:
- parser->flags |= F_CONNECTION_KEEP_ALIVE;
- break;
- case h_connection_close:
- parser->flags |= F_CONNECTION_CLOSE;
- break;
- case h_transfer_encoding_chunked:
- parser->flags |= F_CHUNKED;
- break;
- default:
- break;
- }
-
- parser->state = s_header_field_start;
- goto reexecute_byte;
- }
-
- case s_header_value_discard_ws_almost_done:
- {
- CROW_STRICT_CHECK(ch != CROW_LF);
- parser->state = s_header_value_discard_lws;
- break;
- }
-
- case s_header_value_discard_lws:
- {
- if (ch == ' ' || ch == '\t') {
- parser->state = s_header_value_discard_ws;
- break;
- } else {
- /* header value was empty */
- CROW_MARK(header_value);
- parser->state = s_header_field_start;
- CROW_CALLBACK_DATA_NOADVANCE(header_value);
- goto reexecute_byte;
- }
- }
-
- case s_headers_almost_done:
- {
- CROW_STRICT_CHECK(ch != CROW_LF);
-
- if (parser->flags & F_TRAILING) {
- /* End of a chunked request */
- parser->state = CROW_NEW_MESSAGE();
- CROW_CALLBACK_NOTIFY(message_complete);
- break;
- }
-
- parser->state = s_headers_done;
-
- /* Set this here so that on_headers_complete() callbacks can see it */
- parser->upgrade =
- (parser->flags & F_UPGRADE || parser->method == HTTP_CONNECT);
-
- /* Here we call the headers_complete callback. This is somewhat
- * different than other callbacks because if the user returns 1, we
- * will interpret that as saying that this message has no body. This
- * is needed for the annoying case of recieving a response to a HEAD
- * request.
- *
- * We'd like to use CROW_CALLBACK_NOTIFY_NOADVANCE() here but we cannot, so
- * we have to simulate it by handling a change in errno below.
- */
- if (settings->on_headers_complete) {
- switch (settings->on_headers_complete(parser)) {
- case 0:
- break;
-
- case 1:
- parser->flags |= F_SKIPBODY;
- break;
-
- default:
- CROW_SET_ERRNO(HPE_CB_headers_complete);
- return p - data; /* Error */
- }
- }
-
- if (CROW_HTTP_PARSER_ERRNO(parser) != HPE_OK) {
- return p - data;
- }
-
- goto reexecute_byte;
- }
-
- case s_headers_done:
- {
- CROW_STRICT_CHECK(ch != CROW_LF);
-
- parser->nread = 0;
-
- /* Exit, the rest of the connect is in a different protocol. */
- if (parser->upgrade) {
- parser->state = CROW_NEW_MESSAGE();
- CROW_CALLBACK_NOTIFY(message_complete);
- return (p - data) + 1;
- }
-
- if (parser->flags & F_SKIPBODY) {
- parser->state = CROW_NEW_MESSAGE();
- CROW_CALLBACK_NOTIFY(message_complete);
- } else if (parser->flags & F_CHUNKED) {
- /* chunked encoding - ignore Content-Length header */
- parser->state = s_chunk_size_start;
- } else {
- if (parser->content_length == 0) {
- /* Content-Length header given but zero: Content-Length: 0\r\n */
- parser->state = CROW_NEW_MESSAGE();
- CROW_CALLBACK_NOTIFY(message_complete);
- } else if (parser->content_length != CROW_ULLONG_MAX) {
- /* Content-Length header given and non-zero */
- parser->state = s_body_identity;
- } else {
- if (parser->type == HTTP_REQUEST ||
- !http_message_needs_eof(parser)) {
- /* Assume content-length 0 - read the next */
- parser->state = CROW_NEW_MESSAGE();
- CROW_CALLBACK_NOTIFY(message_complete);
- } else {
- /* Read body until EOF */
- parser->state = s_body_identity_eof;
- }
- }
- }
-
- break;
- }
-
- case s_body_identity:
- {
- uint64_t to_read = CROW_MIN(parser->content_length,
- (uint64_t) ((data + len) - p));
-
- assert(parser->content_length != 0
- && parser->content_length != CROW_ULLONG_MAX);
-
- /* The difference between advancing content_length and p is because
- * the latter will automaticaly advance on the next loop iteration.
- * Further, if content_length ends up at 0, we want to see the last
- * byte again for our message complete callback.
- */
- CROW_MARK(body);
- parser->content_length -= to_read;
- p += to_read - 1;
-
- if (parser->content_length == 0) {
- parser->state = s_message_done;
-
- /* Mimic CROW_CALLBACK_DATA_NOADVANCE() but with one extra byte.
- *
- * The alternative to doing this is to wait for the next byte to
- * trigger the data callback, just as in every other case. The
- * problem with this is that this makes it difficult for the test
- * harness to distinguish between complete-on-EOF and
- * complete-on-length. It's not clear that this distinction is
- * important for applications, but let's keep it for now.
- */
- CROW_CALLBACK_DATA_(body, p - body_mark + 1, p - data);
- goto reexecute_byte;
- }
-
- break;
- }
-
- /* read until EOF */
- case s_body_identity_eof:
- CROW_MARK(body);
- p = data + len - 1;
-
- break;
-
- case s_message_done:
- parser->state = CROW_NEW_MESSAGE();
- CROW_CALLBACK_NOTIFY(message_complete);
- break;
-
- case s_chunk_size_start:
- {
- assert(parser->nread == 1);
- assert(parser->flags & F_CHUNKED);
-
- unhex_val = unhex[(unsigned char)ch];
- if (unhex_val == -1) {
- CROW_SET_ERRNO(HPE_INVALID_CHUNK_SIZE);
- goto error;
- }
-
- parser->content_length = unhex_val;
- parser->state = s_chunk_size;
- break;
- }
-
- case s_chunk_size:
- {
- uint64_t t;
-
- assert(parser->flags & F_CHUNKED);
-
- if (ch == CROW_CR) {
- parser->state = s_chunk_size_almost_done;
- break;
- }
-
- unhex_val = unhex[(unsigned char)ch];
-
- if (unhex_val == -1) {
- if (ch == ';' || ch == ' ') {
- parser->state = s_chunk_parameters;
- break;
- }
-
- CROW_SET_ERRNO(HPE_INVALID_CHUNK_SIZE);
- goto error;
- }
-
- t = parser->content_length;
- t *= 16;
- t += unhex_val;
-
- /* Overflow? Test against a conservative limit for simplicity. */
- if ((CROW_ULLONG_MAX - 16) / 16 < parser->content_length) {
- CROW_SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
- goto error;
- }
-
- parser->content_length = t;
- break;
- }
-
- case s_chunk_parameters:
- {
- assert(parser->flags & F_CHUNKED);
- /* just ignore this shit. TODO check for overflow */
- if (ch == CROW_CR) {
- parser->state = s_chunk_size_almost_done;
- break;
- }
- break;
- }
-
- case s_chunk_size_almost_done:
- {
- assert(parser->flags & F_CHUNKED);
- CROW_STRICT_CHECK(ch != CROW_LF);
-
- parser->nread = 0;
-
- if (parser->content_length == 0) {
- parser->flags |= F_TRAILING;
- parser->state = s_header_field_start;
- } else {
- parser->state = s_chunk_data;
- }
- break;
- }
-
- case s_chunk_data:
- {
- uint64_t to_read = CROW_MIN(parser->content_length,
- (uint64_t) ((data + len) - p));
-
- assert(parser->flags & F_CHUNKED);
- assert(parser->content_length != 0
- && parser->content_length != CROW_ULLONG_MAX);
-
- /* See the explanation in s_body_identity for why the content
- * length and data pointers are managed this way.
- */
- CROW_MARK(body);
- parser->content_length -= to_read;
- p += to_read - 1;
-
- if (parser->content_length == 0) {
- parser->state = s_chunk_data_almost_done;
- }
-
- break;
- }
-
- case s_chunk_data_almost_done:
- assert(parser->flags & F_CHUNKED);
- assert(parser->content_length == 0);
- CROW_STRICT_CHECK(ch != CROW_CR);
- parser->state = s_chunk_data_done;
- CROW_CALLBACK_DATA(body);
- break;
-
- case s_chunk_data_done:
- assert(parser->flags & F_CHUNKED);
- CROW_STRICT_CHECK(ch != CROW_LF);
- parser->nread = 0;
- parser->state = s_chunk_size_start;
- break;
-
- default:
- assert(0 && "unhandled state");
- CROW_SET_ERRNO(HPE_INVALID_INTERNAL_STATE);
- goto error;
- }
- }
-
- /* Run callbacks for any marks that we have leftover after we ran our of
- * bytes. There should be at most one of these set, so it's OK to invoke
- * them in series (unset marks will not result in callbacks).
- *
- * We use the NOADVANCE() variety of callbacks here because 'p' has already
- * overflowed 'data' and this allows us to correct for the off-by-one that
- * we'd otherwise have (since CROW_CALLBACK_DATA() is meant to be run with a 'p'
- * value that's in-bounds).
- */
-
- assert(((header_field_mark ? 1 : 0) +
- (header_value_mark ? 1 : 0) +
- (url_mark ? 1 : 0) +
- (body_mark ? 1 : 0) +
- (status_mark ? 1 : 0)) <= 1);
-
- CROW_CALLBACK_DATA_NOADVANCE(header_field);
- CROW_CALLBACK_DATA_NOADVANCE(header_value);
- CROW_CALLBACK_DATA_NOADVANCE(url);
- CROW_CALLBACK_DATA_NOADVANCE(body);
- CROW_CALLBACK_DATA_NOADVANCE(status);
-
- return len;
-
-error:
- if (CROW_HTTP_PARSER_ERRNO(parser) == HPE_OK) {
- CROW_SET_ERRNO(HPE_UNKNOWN);
- }
-
- return (p - data);
-}
-
-
-/* Does the parser need to see an EOF to find the end of the message? */
-inline int
-http_message_needs_eof (const http_parser *parser)
-{
- if (parser->type == HTTP_REQUEST) {
- return 0;
- }
-
- /* See RFC 2616 section 4.4 */
- if (parser->status_code / 100 == 1 || /* 1xx e.g. Continue */
- parser->status_code == 204 || /* No Content */
- parser->status_code == 304 || /* Not Modified */
- parser->flags & F_SKIPBODY) { /* response to a HEAD request */
- return 0;
- }
-
- if ((parser->flags & F_CHUNKED) || parser->content_length != CROW_ULLONG_MAX) {
- return 0;
- }
-
- return 1;
-}
-
-
-inline int
-http_should_keep_alive (const http_parser *parser)
-{
- if (parser->http_major > 0 && parser->http_minor > 0) {
- /* HTTP/1.1 */
- if (parser->flags & F_CONNECTION_CLOSE) {
- return 0;
- }
- } else {
- /* HTTP/1.0 or earlier */
- if (!(parser->flags & F_CONNECTION_KEEP_ALIVE)) {
- return 0;
- }
- }
-
- return !http_message_needs_eof(parser);
-}
-
-
-inline const char *
-http_method_str (enum http_method m)
-{
-static const char *method_strings[] =
- {
-#define CROW_XX(num, name, string) #string,
- CROW_HTTP_METHOD_MAP(CROW_XX)
-#undef CROW_XX
- };
- return CROW_ELEM_AT(method_strings, m, "");
-}
-
-
-inline void
-http_parser_init (http_parser *parser, enum http_parser_type t)
-{
- void *data = parser->data; /* preserve application data */
- memset(parser, 0, sizeof(*parser));
- parser->data = data;
- parser->type = t;
- parser->state = (t == HTTP_REQUEST ? s_start_req : (t == HTTP_RESPONSE ? s_start_res : s_start_req_or_res));
- parser->http_errno = HPE_OK;
-}
-
-inline const char *
-http_errno_name(enum http_errno err) {
-/* Map errno values to strings for human-readable output */
-#define CROW_HTTP_STRERROR_GEN(n, s) { "HPE_" #n, s },
-static struct {
- const char *name;
- const char *description;
-} http_strerror_tab[] = {
- CROW_HTTP_ERRNO_MAP(CROW_HTTP_STRERROR_GEN)
-};
-#undef CROW_HTTP_STRERROR_GEN
- assert(err < (sizeof(http_strerror_tab)/sizeof(http_strerror_tab[0])));
- return http_strerror_tab[err].name;
-}
-
-inline const char *
-http_errno_description(enum http_errno err) {
-/* Map errno values to strings for human-readable output */
-#define CROW_HTTP_STRERROR_GEN(n, s) { "HPE_" #n, s },
-static struct {
- const char *name;
- const char *description;
-} http_strerror_tab[] = {
- CROW_HTTP_ERRNO_MAP(CROW_HTTP_STRERROR_GEN)
-};
-#undef CROW_HTTP_STRERROR_GEN
- assert(err < (sizeof(http_strerror_tab)/sizeof(http_strerror_tab[0])));
- return http_strerror_tab[err].description;
-}
-
-inline static enum http_host_state
-http_parse_host_char(enum http_host_state s, const char ch) {
- switch(s) {
- case s_http_userinfo:
- case s_http_userinfo_start:
- if (ch == '@') {
- return s_http_host_start;
- }
-
- if (CROW_IS_USERINFO_CHAR(ch)) {
- return s_http_userinfo;
- }
- break;
-
- case s_http_host_start:
- if (ch == '[') {
- return s_http_host_v6_start;
- }
-
- if (CROW_IS_HOST_CHAR(ch)) {
- return s_http_host;
- }
-
- break;
-
- case s_http_host:
- if (CROW_IS_HOST_CHAR(ch)) {
- return s_http_host;
- }
-
- /* FALLTHROUGH */
- case s_http_host_v6_end:
- if (ch == ':') {
- return s_http_host_port_start;
- }
-
- break;
-
- case s_http_host_v6:
- if (ch == ']') {
- return s_http_host_v6_end;
- }
-
- /* FALLTHROUGH */
- case s_http_host_v6_start:
- if (CROW_IS_HEX(ch) || ch == ':' || ch == '.') {
- return s_http_host_v6;
- }
-
- break;
-
- case s_http_host_port:
- case s_http_host_port_start:
- if (CROW_IS_NUM(ch)) {
- return s_http_host_port;
- }
-
- break;
-
- default:
- break;
- }
- return s_http_host_dead;
-}
-
-inline int
-http_parse_host(const char * buf, struct http_parser_url *u, int found_at) {
- enum http_host_state s;
-
- const char *p;
- size_t buflen = u->field_data[UF_HOST].off + u->field_data[UF_HOST].len;
-
- u->field_data[UF_HOST].len = 0;
-
- s = found_at ? s_http_userinfo_start : s_http_host_start;
-
- for (p = buf + u->field_data[UF_HOST].off; p < buf + buflen; p++) {
- enum http_host_state new_s = http_parse_host_char(s, *p);
-
- if (new_s == s_http_host_dead) {
- return 1;
- }
-
- switch(new_s) {
- case s_http_host:
- if (s != s_http_host) {
- u->field_data[UF_HOST].off = p - buf;
- }
- u->field_data[UF_HOST].len++;
- break;
-
- case s_http_host_v6:
- if (s != s_http_host_v6) {
- u->field_data[UF_HOST].off = p - buf;
- }
- u->field_data[UF_HOST].len++;
- break;
-
- case s_http_host_port:
- if (s != s_http_host_port) {
- u->field_data[UF_PORT].off = p - buf;
- u->field_data[UF_PORT].len = 0;
- u->field_set |= (1 << UF_PORT);
- }
- u->field_data[UF_PORT].len++;
- break;
-
- case s_http_userinfo:
- if (s != s_http_userinfo) {
- u->field_data[UF_USERINFO].off = p - buf ;
- u->field_data[UF_USERINFO].len = 0;
- u->field_set |= (1 << UF_USERINFO);
- }
- u->field_data[UF_USERINFO].len++;
- break;
-
- default:
- break;
- }
- s = new_s;
- }
-
- /* Make sure we don't end somewhere unexpected */
- switch (s) {
- case s_http_host_start:
- case s_http_host_v6_start:
- case s_http_host_v6:
- case s_http_host_port_start:
- case s_http_userinfo:
- case s_http_userinfo_start:
- return 1;
- default:
- break;
- }
-
- return 0;
-}
-
-inline int
-http_parser_parse_url(const char *buf, size_t buflen, int is_connect,
- struct http_parser_url *u)
-{
- enum state s;
- const char *p;
- enum http_parser_url_fields uf, old_uf;
- int found_at = 0;
-
- u->port = u->field_set = 0;
- s = is_connect ? s_req_server_start : s_req_spaces_before_url;
- old_uf = UF_MAX;
-
- for (p = buf; p < buf + buflen; p++) {
- s = parse_url_char(s, *p);
-
- /* Figure out the next field that we're operating on */
- switch (s) {
- case s_dead:
- return 1;
-
- /* Skip delimeters */
- case s_req_schema_slash:
- case s_req_schema_slash_slash:
- case s_req_server_start:
- case s_req_query_string_start:
- case s_req_fragment_start:
- continue;
-
- case s_req_schema:
- uf = UF_SCHEMA;
- break;
-
- case s_req_server_with_at:
- found_at = 1;
-
- /* FALLTROUGH */
- case s_req_server:
- uf = UF_HOST;
- break;
-
- case s_req_path:
- uf = UF_PATH;
- break;
-
- case s_req_query_string:
- uf = UF_QUERY;
- break;
-
- case s_req_fragment:
- uf = UF_FRAGMENT;
- break;
-
- default:
- assert(!"Unexpected state");
- return 1;
- }
-
- /* Nothing's changed; soldier on */
- if (uf == old_uf) {
- u->field_data[uf].len++;
- continue;
- }
-
- u->field_data[uf].off = p - buf;
- u->field_data[uf].len = 1;
-
- u->field_set |= (1 << uf);
- old_uf = uf;
- }
-
- /* host must be present if there is a schema */
- /* parsing http:///toto will fail */
- if ((u->field_set & ((1 << UF_SCHEMA) | (1 << UF_HOST))) != 0) {
- if (http_parse_host(buf, u, found_at) != 0) {
- return 1;
- }
- }
-
- /* CONNECT requests can only contain "hostname:port" */
- if (is_connect && u->field_set != ((1 << UF_HOST)|(1 << UF_PORT))) {
- return 1;
- }
-
- if (u->field_set & (1 << UF_PORT)) {
- /* Don't bother with endp; we've already validated the string */
- unsigned long v = strtoul(buf + u->field_data[UF_PORT].off, NULL, 10);
-
- /* Ports have a max value of 2^16 */
- if (v > 0xffff) {
- return 1;
- }
-
- u->port = (uint16_t) v;
- }
-
- return 0;
-}
-
-inline void
-http_parser_pause(http_parser *parser, int paused) {
- /* Users should only be pausing/unpausing a parser that is not in an error
- * state. In non-debug builds, there's not much that we can do about this
- * other than ignore it.
- */
- if (CROW_HTTP_PARSER_ERRNO(parser) == HPE_OK ||
- CROW_HTTP_PARSER_ERRNO(parser) == HPE_PAUSED) {
- CROW_SET_ERRNO((paused) ? HPE_PAUSED : HPE_OK);
- } else {
- assert(0 && "Attempting to pause parser in error state");
- }
-}
-
-inline int
-http_body_is_final(const struct http_parser *parser) {
- return parser->state == s_message_done;
-}
-
-inline unsigned long
-http_parser_version(void) {
- return CROW_HTTP_PARSER_VERSION_MAJOR * 0x10000 |
- CROW_HTTP_PARSER_VERSION_MINOR * 0x00100 |
- CROW_HTTP_PARSER_VERSION_PATCH * 0x00001;
-}
-
-#undef CROW_HTTP_METHOD_MAP
-#undef CROW_HTTP_ERRNO_MAP
-#undef CROW_SET_ERRNO
-#undef CROW_CALLBACK_NOTIFY_
-#undef CROW_CALLBACK_NOTIFY
-#undef CROW_CALLBACK_NOTIFY_NOADVANCE
-#undef CROW_CALLBACK_DATA_
-#undef CROW_CALLBACK_DATA
-#undef CROW_CALLBACK_DATA_NOADVANCE
-#undef CROW_MARK
-#undef CROW_PROXY_CONNECTION
-#undef CROW_CONNECTION
-#undef CROW_CONTENT_LENGTH
-#undef CROW_TRANSFER_ENCODING
-#undef CROW_UPGRADE
-#undef CROW_CHUNKED
-#undef CROW_KEEP_ALIVE
-#undef CROW_CLOSE
-#undef CROW_PARSING_HEADER
-#undef CROW_CR
-#undef CROW_LF
-#undef CROW_LOWER
-#undef CROW_IS_ALPHA
-#undef CROW_IS_NUM
-#undef CROW_IS_ALPHANUM
-#undef CROW_IS_HEX
-#undef CROW_IS_MARK
-#undef CROW_IS_USERINFO_CHAR
-#undef CROW_TOKEN
-#undef CROW_IS_URL_CHAR
-#undef CROW_IS_HOST_CHAR
-#undef CROW_start_state
-#undef CROW_STRICT_CHECK
-#undef CROW_NEW_MESSAGE
-
-#ifdef __cplusplus
-}
-#endif
-#endif
-
-
-
-#pragma once
-
-#include
-#include
-#include
-
-namespace crow
-{
- struct ci_hash
- {
- size_t operator()(const std::string& key) const
- {
- std::size_t seed = 0;
- std::locale locale;
-
- for(auto c : key)
- {
- boost::hash_combine(seed, std::toupper(c, locale));
- }
-
- return seed;
- }
- };
-
- struct ci_key_eq
- {
- bool operator()(const std::string& l, const std::string& r) const
- {
- return boost::iequals(l, r);
- }
- };
-
- using ci_map = std::unordered_multimap;
-}
-
-
-
-/*
- *
- * TinySHA1 - a header only implementation of the SHA1 algorithm in C++. Based
- * on the implementation in boost::uuid::details.
- *
- * SHA1 Wikipedia Page: http://en.wikipedia.org/wiki/SHA-1
- *
- * Copyright (c) 2012-22 SAURAV MOHAPATRA
- *
- * Permission to use, copy, modify, and distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
- * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
- * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
- * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
- * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
- */
-#ifndef _TINY_SHA1_HPP_
-#define _TINY_SHA1_HPP_
-#include
-#include
-#include
-#include
-namespace sha1
-{
- class SHA1
- {
- public:
- typedef uint32_t digest32_t[5];
- typedef uint8_t digest8_t[20];
- inline static uint32_t LeftRotate(uint32_t value, size_t count) {
- return (value << count) ^ (value >> (32-count));
- }
- SHA1(){ reset(); }
- virtual ~SHA1() {}
- SHA1(const SHA1& s) { *this = s; }
- const SHA1& operator = (const SHA1& s) {
- memcpy(m_digest, s.m_digest, 5 * sizeof(uint32_t));
- memcpy(m_block, s.m_block, 64);
- m_blockByteIndex = s.m_blockByteIndex;
- m_byteCount = s.m_byteCount;
- return *this;
- }
- SHA1& reset() {
- m_digest[0] = 0x67452301;
- m_digest[1] = 0xEFCDAB89;
- m_digest[2] = 0x98BADCFE;
- m_digest[3] = 0x10325476;
- m_digest[4] = 0xC3D2E1F0;
- m_blockByteIndex = 0;
- m_byteCount = 0;
- return *this;
- }
- SHA1& processByte(uint8_t octet) {
- this->m_block[this->m_blockByteIndex++] = octet;
- ++this->m_byteCount;
- if(m_blockByteIndex == 64) {
- this->m_blockByteIndex = 0;
- processBlock();
- }
- return *this;
- }
- SHA1& processBlock(const void* const start, const void* const end) {
- const uint8_t* begin = static_cast(start);
- const uint8_t* finish = static_cast(end);
- while(begin != finish) {
- processByte(*begin);
- begin++;
- }
- return *this;
- }
- SHA1& processBytes(const void* const data, size_t len) {
- const uint8_t* block = static_cast(data);
- processBlock(block, block + len);
- return *this;
- }
- const uint32_t* getDigest(digest32_t digest) {
- size_t bitCount = this->m_byteCount * 8;
- processByte(0x80);
- if (this->m_blockByteIndex > 56) {
- while (m_blockByteIndex != 0) {
- processByte(0);
- }
- while (m_blockByteIndex < 56) {
- processByte(0);
- }
- } else {
- while (m_blockByteIndex < 56) {
- processByte(0);
- }
- }
- processByte(0);
- processByte(0);
- processByte(0);
- processByte(0);
- processByte( static_cast((bitCount>>24) & 0xFF));
- processByte( static_cast((bitCount>>16) & 0xFF));
- processByte( static_cast((bitCount>>8 ) & 0xFF));
- processByte( static_cast((bitCount) & 0xFF));
-
- memcpy(digest, m_digest, 5 * sizeof(uint32_t));
- return digest;
- }
- const uint8_t* getDigestBytes(digest8_t digest) {
- digest32_t d32;
- getDigest(d32);
- size_t di = 0;
- digest[di++] = ((d32[0] >> 24) & 0xFF);
- digest[di++] = ((d32[0] >> 16) & 0xFF);
- digest[di++] = ((d32[0] >> 8) & 0xFF);
- digest[di++] = ((d32[0]) & 0xFF);
-
- digest[di++] = ((d32[1] >> 24) & 0xFF);
- digest[di++] = ((d32[1] >> 16) & 0xFF);
- digest[di++] = ((d32[1] >> 8) & 0xFF);
- digest[di++] = ((d32[1]) & 0xFF);
-
- digest[di++] = ((d32[2] >> 24) & 0xFF);
- digest[di++] = ((d32[2] >> 16) & 0xFF);
- digest[di++] = ((d32[2] >> 8) & 0xFF);
- digest[di++] = ((d32[2]) & 0xFF);
-
- digest[di++] = ((d32[3] >> 24) & 0xFF);
- digest[di++] = ((d32[3] >> 16) & 0xFF);
- digest[di++] = ((d32[3] >> 8) & 0xFF);
- digest[di++] = ((d32[3]) & 0xFF);
-
- digest[di++] = ((d32[4] >> 24) & 0xFF);
- digest[di++] = ((d32[4] >> 16) & 0xFF);
- digest[di++] = ((d32[4] >> 8) & 0xFF);
- digest[di++] = ((d32[4]) & 0xFF);
- return digest;
- }
-
- protected:
- void processBlock() {
- uint32_t w[80];
- for (size_t i = 0; i < 16; i++) {
- w[i] = (m_block[i*4 + 0] << 24);
- w[i] |= (m_block[i*4 + 1] << 16);
- w[i] |= (m_block[i*4 + 2] << 8);
- w[i] |= (m_block[i*4 + 3]);
- }
- for (size_t i = 16; i < 80; i++) {
- w[i] = LeftRotate((w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16]), 1);
- }
-
- uint32_t a = m_digest[0];
- uint32_t b = m_digest[1];
- uint32_t c = m_digest[2];
- uint32_t d = m_digest[3];
- uint32_t e = m_digest[4];
-
- for (std::size_t i=0; i<80; ++i) {
- uint32_t f = 0;
- uint32_t k = 0;
-
- if (i<20) {
- f = (b & c) | (~b & d);
- k = 0x5A827999;
- } else if (i<40) {
- f = b ^ c ^ d;
- k = 0x6ED9EBA1;
- } else if (i<60) {
- f = (b & c) | (b & d) | (c & d);
- k = 0x8F1BBCDC;
- } else {
- f = b ^ c ^ d;
- k = 0xCA62C1D6;
- }
- uint32_t temp = LeftRotate(a, 5) + f + e + k + w[i];
- e = d;
- d = c;
- c = LeftRotate(b, 30);
- b = a;
- a = temp;
- }
-
- m_digest[0] += a;
- m_digest[1] += b;
- m_digest[2] += c;
- m_digest[3] += d;
- m_digest[4] += e;
- }
- private:
- digest32_t m_digest;
- uint8_t m_block[64];
- size_t m_blockByteIndex;
- size_t m_byteCount;
- };
-}
-#endif
-
-
-
-#pragma once
-// settings for crow
-// TODO - replace with runtime config. libucl?
-
-/* #ifdef - enables debug mode */
-//#define CROW_ENABLE_DEBUG
-
-/* #ifdef - enables logging */
-#define CROW_ENABLE_LOGGING
-
-/* #ifdef - enables ssl */
-//#define CROW_ENABLE_SSL
-
-/* #define - specifies log level */
-/*
- Debug = 0
- Info = 1
- Warning = 2
- Error = 3
- Critical = 4
-
- default to INFO
-*/
-#define CROW_LOG_LEVEL 1
-
-
-// compiler flags
-#if __cplusplus >= 201402L
-#define CROW_CAN_USE_CPP14
-#endif
-
-#if defined(_MSC_VER)
-#if _MSC_VER < 1900
-#define CROW_MSVC_WORKAROUND
-#define constexpr const
-#define noexcept throw()
-#endif
-#endif
-
-
-
-#pragma once
-#include
-#ifdef CROW_ENABLE_SSL
-#include
-#endif
-
-
-namespace crow
-{
- using namespace boost;
- using tcp = asio::ip::tcp;
-
- struct SocketAdaptor
- {
- using context = void;
- SocketAdaptor(boost::asio::io_service& io_service, context*)
- : socket_(io_service)
- {
- }
-
- boost::asio::io_service& get_io_service()
- {
- return socket_.get_io_service();
- }
-
- tcp::socket& raw_socket()
- {
- return socket_;
- }
-
- tcp::socket& socket()
- {
- return socket_;
- }
-
- tcp::endpoint remote_endpoint()
- {
- return socket_.remote_endpoint();
- }
-
- bool is_open()
- {
- return socket_.is_open();
- }
-
- void close()
- {
- socket_.close();
- }
-
- template
- void start(F f)
- {
- f(boost::system::error_code());
- }
-
- tcp::socket socket_;
- };
-
-#ifdef CROW_ENABLE_SSL
- struct SSLAdaptor
- {
- using context = boost::asio::ssl::context;
- using ssl_socket_t = boost::asio::ssl::stream;
- SSLAdaptor(boost::asio::io_service& io_service, context* ctx)
- : ssl_socket_(new ssl_socket_t(io_service, *ctx))
- {
- }
-
- boost::asio::ssl::stream& socket()
- {
- return *ssl_socket_;
- }
-
- tcp::socket::lowest_layer_type&
- raw_socket()
- {
- return ssl_socket_->lowest_layer();
- }
-
- tcp::endpoint remote_endpoint()
- {
- return raw_socket().remote_endpoint();
- }
-
- bool is_open()
- {
- return raw_socket().is_open();
- }
-
- void close()
- {
- raw_socket().close();
- }
-
- boost::asio::io_service& get_io_service()
- {
- return raw_socket().get_io_service();
- }
-
- template
- void start(F f)
- {
- ssl_socket_->async_handshake(boost::asio::ssl::stream_base::server,
- [f](const boost::system::error_code& ec) {
- f(ec);
- });
- }
-
- std::unique_ptr> ssl_socket_;
- };
-#endif
-}
-
-
-
-#pragma once
-
-//#define CROW_JSON_NO_ERROR_CHECK
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-
-
-
-
-#if defined(__GNUG__) || defined(__clang__)
-#define crow_json_likely(x) __builtin_expect(x, 1)
-#define crow_json_unlikely(x) __builtin_expect(x, 0)
-#else
-#define crow_json_likely(x) x
-#define crow_json_unlikely(x) x
-#endif
-
-
-namespace crow
-{
- namespace mustache
- {
- class template_t;
- }
-
- namespace json
- {
- inline void escape(const std::string& str, std::string& ret)
- {
- ret.reserve(ret.size() + str.size()+str.size()/4);
- for(char c:str)
- {
- switch(c)
- {
- case '"': ret += "\\\""; break;
- case '\\': ret += "\\\\"; break;
- case '\n': ret += "\\n"; break;
- case '\b': ret += "\\b"; break;
- case '\f': ret += "\\f"; break;
- case '\r': ret += "\\r"; break;
- case '\t': ret += "\\t"; break;
- default:
- if (0 <= c && c < 0x20)
- {
- ret += "\\u00";
- auto to_hex = [](char c)
- {
- c = c&0xf;
- if (c < 10)
- return '0' + c;
- return 'a'+c-10;
- };
- ret += to_hex(c/16);
- ret += to_hex(c%16);
- }
- else
- ret += c;
- break;
- }
- }
- }
- inline std::string escape(const std::string& str)
- {
- std::string ret;
- escape(str, ret);
- return ret;
- }
-
- enum class type : char
- {
- Null,
- False,
- True,
- Number,
- String,
- List,
- Object,
- };
-
- inline const char* get_type_str(type t) {
- switch(t){
- case type::Number: return "Number";
- case type::False: return "False";
- case type::True: return "True";
- case type::List: return "List";
- case type::String: return "String";
- case type::Object: return "Object";
- default: return "Unknown";
- }
- }
-
- class rvalue;
- rvalue load(const char* data, size_t size);
-
- namespace detail
- {
-
- struct r_string
- : boost::less_than_comparable,
- boost::less_than_comparable,
- boost::equality_comparable,
- boost::equality_comparable
- {
- r_string() {};
- r_string(char* s, char* e)
- : s_(s), e_(e)
- {};
- ~r_string()
- {
- if (owned_)
- delete[] s_;
- }
-
- r_string(const r_string& r)
- {
- *this = r;
- }
-
- r_string(r_string&& r)
- {
- *this = r;
- }
-
- r_string& operator = (r_string&& r)
- {
- s_ = r.s_;
- e_ = r.e_;
- owned_ = r.owned_;
- if (r.owned_)
- r.owned_ = 0;
- return *this;
- }
-
- r_string& operator = (const r_string& r)
- {
- s_ = r.s_;
- e_ = r.e_;
- owned_ = 0;
- return *this;
- }
-
- operator std::string () const
- {
- return std::string(s_, e_);
- }
-
-
- const char* begin() const { return s_; }
- const char* end() const { return e_; }
- size_t size() const { return end() - begin(); }
-
- using iterator = const char*;
- using const_iterator = const char*;
-
- char* s_;
- mutable char* e_;
- uint8_t owned_{0};
- friend std::ostream& operator << (std::ostream& os, const r_string& s)
- {
- os << (std::string)s;
- return os;
- }
- private:
- void force(char* s, uint32_t /*length*/)
- {
- s_ = s;
- owned_ = 1;
- }
- friend rvalue crow::json::load(const char* data, size_t size);
- };
-
- inline bool operator < (const r_string& l, const r_string& r)
- {
- return boost::lexicographical_compare(l,r);
- }
-
- inline bool operator < (const r_string& l, const std::string& r)
- {
- return boost::lexicographical_compare(l,r);
- }
-
- inline bool operator > (const r_string& l, const std::string& r)
- {
- return boost::lexicographical_compare(r,l);
- }
-
- inline bool operator == (const r_string& l, const r_string& r)
- {
- return boost::equals(l,r);
- }
-
- inline bool operator == (const r_string& l, const std::string& r)
- {
- return boost::equals(l,r);
- }
- }
-
- class rvalue
- {
- static const int cached_bit = 2;
- static const int error_bit = 4;
- public:
- rvalue() noexcept : option_{error_bit}
- {}
- rvalue(type t) noexcept
- : lsize_{}, lremain_{}, t_{t}
- {}
- rvalue(type t, char* s, char* e) noexcept
- : start_{s},
- end_{e},
- t_{t}
- {}
-
- rvalue(const rvalue& r)
- : start_(r.start_),
- end_(r.end_),
- key_(r.key_),
- t_(r.t_),
- option_(r.option_)
- {
- copy_l(r);
- }
-
- rvalue(rvalue&& r) noexcept
- {
- *this = std::move(r);
- }
-
- rvalue& operator = (const rvalue& r)
- {
- start_ = r.start_;
- end_ = r.end_;
- key_ = r.key_;
- copy_l(r);
- t_ = r.t_;
- option_ = r.option_;
- return *this;
- }
- rvalue& operator = (rvalue&& r) noexcept
- {
- start_ = r.start_;
- end_ = r.end_;
- key_ = std::move(r.key_);
- l_ = std::move(r.l_);
- lsize_ = r.lsize_;
- lremain_ = r.lremain_;
- t_ = r.t_;
- option_ = r.option_;
- return *this;
- }
-
- explicit operator bool() const noexcept
- {
- return (option_ & error_bit) == 0;
- }
-
- explicit operator int64_t() const
- {
- return i();
- }
-
- explicit operator uint64_t() const
- {
- return u();
- }
-
- explicit operator int() const
- {
- return (int)i();
- }
-
- type t() const
- {
-#ifndef CROW_JSON_NO_ERROR_CHECK
- if (option_ & error_bit)
- {
- throw std::runtime_error("invalid json object");
- }
-#endif
- return t_;
- }
-
- int64_t i() const
- {
-#ifndef CROW_JSON_NO_ERROR_CHECK
- switch (t()) {
- case type::Number:
- case type::String:
- return boost::lexical_cast(start_, end_-start_);
- default:
- const std::string msg = "expected number, got: "
- + std::string(get_type_str(t()));
- throw std::runtime_error(msg);
- }
-#endif
- return boost::lexical_cast(start_, end_-start_);
- }
-
- uint64_t u() const
- {
-#ifndef CROW_JSON_NO_ERROR_CHECK
- switch (t()) {
- case type::Number:
- case type::String:
- return boost::lexical_cast(start_, end_-start_);
- default:
- throw std::runtime_error(std::string("expected number, got: ") + get_type_str(t()));
- }
-#endif
- return boost::lexical_cast(start_, end_-start_);
- }
-
- double d() const
- {
-#ifndef CROW_JSON_NO_ERROR_CHECK
- if (t() != type::Number)
- throw std::runtime_error("value is not number");
-#endif
- return boost::lexical_cast(start_, end_-start_);
- }
-
- bool b() const
- {
-#ifndef CROW_JSON_NO_ERROR_CHECK
- if (t() != type::True && t() != type::False)
- throw std::runtime_error("value is not boolean");
-#endif
- return t() == type::True;
- }
-
- void unescape() const
- {
- if (*(start_-1))
- {
- char* head = start_;
- char* tail = start_;
- while(head != end_)
- {
- if (*head == '\\')
- {
- switch(*++head)
- {
- case '"': *tail++ = '"'; break;
- case '\\': *tail++ = '\\'; break;
- case '/': *tail++ = '/'; break;
- case 'b': *tail++ = '\b'; break;
- case 'f': *tail++ = '\f'; break;
- case 'n': *tail++ = '\n'; break;
- case 'r': *tail++ = '\r'; break;
- case 't': *tail++ = '\t'; break;
- case 'u':
- {
- auto from_hex = [](char c)
- {
- if (c >= 'a')
- return c - 'a' + 10;
- if (c >= 'A')
- return c - 'A' + 10;
- return c - '0';
- };
- unsigned int code =
- (from_hex(head[1])<<12) +
- (from_hex(head[2])<< 8) +
- (from_hex(head[3])<< 4) +
- from_hex(head[4]);
- if (code >= 0x800)
- {
- *tail++ = 0xE0 | (code >> 12);
- *tail++ = 0x80 | ((code >> 6) & 0x3F);
- *tail++ = 0x80 | (code & 0x3F);
- }
- else if (code >= 0x80)
- {
- *tail++ = 0xC0 | (code >> 6);
- *tail++ = 0x80 | (code & 0x3F);
- }
- else
- {
- *tail++ = code;
- }
- head += 4;
- }
- break;
- }
- }
- else
- *tail++ = *head;
- head++;
- }
- end_ = tail;
- *end_ = 0;
- *(start_-1) = 0;
- }
- }
-
- detail::r_string s() const
- {
-#ifndef CROW_JSON_NO_ERROR_CHECK
- if (t() != type::String)
- throw std::runtime_error("value is not string");
-#endif
- unescape();
- return detail::r_string{start_, end_};
- }
-
- bool has(const char* str) const
- {
- return has(std::string(str));
- }
-
- bool has(const std::string& str) const
- {
- struct Pred
- {
- bool operator()(const rvalue& l, const rvalue& r) const
- {
- return l.key_ < r.key_;
- };
- bool operator()(const rvalue& l, const std::string& r) const
- {
- return l.key_ < r;
- };
- bool operator()(const std::string& l, const rvalue& r) const
- {
- return l < r.key_;
- };
- };
- if (!is_cached())
- {
- std::sort(begin(), end(), Pred());
- set_cached();
- }
- auto it = lower_bound(begin(), end(), str, Pred());
- return it != end() && it->key_ == str;
- }
-
- int count(const std::string& str)
- {
- return has(str) ? 1 : 0;
- }
-
- rvalue* begin() const
- {
-#ifndef CROW_JSON_NO_ERROR_CHECK
- if (t() != type::Object && t() != type::List)
- throw std::runtime_error("value is not a container");
-#endif
- return l_.get();
- }
- rvalue* end() const
- {
-#ifndef CROW_JSON_NO_ERROR_CHECK
- if (t() != type::Object && t() != type::List)
- throw std::runtime_error("value is not a container");
-#endif
- return l_.get()+lsize_;
- }
-
- const detail::r_string& key() const
- {
- return key_;
- }
-
- size_t size() const
- {
- if (t() == type::String)
- return s().size();
-#ifndef CROW_JSON_NO_ERROR_CHECK
- if (t() != type::Object && t() != type::List)
- throw std::runtime_error("value is not a container");
-#endif
- return lsize_;
- }
-
- const rvalue& operator[](int index) const
- {
-#ifndef CROW_JSON_NO_ERROR_CHECK
- if (t() != type::List)
- throw std::runtime_error("value is not a list");
- if (index >= (int)lsize_ || index < 0)
- throw std::runtime_error("list out of bound");
-#endif
- return l_[index];
- }
-
- const rvalue& operator[](size_t index) const
- {
-#ifndef CROW_JSON_NO_ERROR_CHECK
- if (t() != type::List)
- throw std::runtime_error("value is not a list");
- if (index >= lsize_)
- throw std::runtime_error("list out of bound");
-#endif
- return l_[index];
- }
-
- const rvalue& operator[](const char* str) const
- {
- return this->operator[](std::string(str));
- }
-
- const rvalue& operator[](const std::string& str) const
- {
-#ifndef CROW_JSON_NO_ERROR_CHECK
- if (t() != type::Object)
- throw std::runtime_error("value is not an object");
-#endif
- struct Pred
- {
- bool operator()(const rvalue& l, const rvalue& r) const
- {
- return l.key_ < r.key_;
- };
- bool operator()(const rvalue& l, const std::string& r) const
- {
- return l.key_ < r;
- };
- bool operator()(const std::string& l, const rvalue& r) const
- {
- return l < r.key_;
- };
- };
- if (!is_cached())
- {
- std::sort(begin(), end(), Pred());
- set_cached();
- }
- auto it = lower_bound(begin(), end(), str, Pred());
- if (it != end() && it->key_ == str)
- return *it;
-#ifndef CROW_JSON_NO_ERROR_CHECK
- throw std::runtime_error("cannot find key");
-#else
- static rvalue nullValue;
- return nullValue;
-#endif
- }
-
- void set_error()
- {
- option_|=error_bit;
- }
-
- bool error() const
- {
- return (option_&error_bit)!=0;
- }
- private:
- bool is_cached() const
- {
- return (option_&cached_bit)!=0;
- }
- void set_cached() const
- {
- option_ |= cached_bit;
- }
- void copy_l(const rvalue& r)
- {
- if (r.t() != type::Object && r.t() != type::List)
- return;
- lsize_ = r.lsize_;
- lremain_ = 0;
- l_.reset(new rvalue[lsize_]);
- std::copy(r.begin(), r.end(), begin());
- }
-
- void emplace_back(rvalue&& v)
- {
- if (!lremain_)
- {
- int new_size = lsize_ + lsize_;
- if (new_size - lsize_ > 60000)
- new_size = lsize_ + 60000;
- if (new_size < 4)
- new_size = 4;
- rvalue* p = new rvalue[new_size];
- rvalue* p2 = p;
- for(auto& x : *this)
- *p2++ = std::move(x);
- l_.reset(p);
- lremain_ = new_size - lsize_;
- }
- l_[lsize_++] = std::move(v);
- lremain_ --;
- }
-
- mutable char* start_;
- mutable char* end_;
- detail::r_string key_;
- std::unique_ptr l_;
- uint32_t lsize_;
- uint16_t lremain_;
- type t_;
- mutable uint8_t option_{0};
-
- friend rvalue load_nocopy_internal(char* data, size_t size);
- friend rvalue load(const char* data, size_t size);
- friend std::ostream& operator <<(std::ostream& os, const rvalue& r)
- {
- switch(r.t_)
- {
-
- case type::Null: os << "null"; break;
- case type::False: os << "false"; break;
- case type::True: os << "true"; break;
- case type::Number: os << r.d(); break;
- case type::String: os << '"' << r.s() << '"'; break;
- case type::List:
- {
- os << '[';
- bool first = true;
- for(auto& x : r)
- {
- if (!first)
- os << ',';
- first = false;
- os << x;
- }
- os << ']';
- }
- break;
- case type::Object:
- {
- os << '{';
- bool first = true;
- for(auto& x : r)
- {
- if (!first)
- os << ',';
- os << '"' << escape(x.key_) << "\":";
- first = false;
- os << x;
- }
- os << '}';
- }
- break;
- }
- return os;
- }
- };
- namespace detail {
- }
-
- inline bool operator == (const rvalue& l, const std::string& r)
- {
- return l.s() == r;
- }
-
- inline bool operator == (const std::string& l, const rvalue& r)
- {
- return l == r.s();
- }
-
- inline bool operator != (const rvalue& l, const std::string& r)
- {
- return l.s() != r;
- }
-
- inline bool operator != (const std::string& l, const rvalue& r)
- {
- return l != r.s();
- }
-
- inline bool operator == (const rvalue& l, double r)
- {
- return l.d() == r;
- }
-
- inline bool operator == (double l, const rvalue& r)
- {
- return l == r.d();
- }
-
- inline bool operator != (const rvalue& l, double r)
- {
- return l.d() != r;
- }
-
- inline bool operator != (double l, const rvalue& r)
- {
- return l != r.d();
- }
-
-
- inline rvalue load_nocopy_internal(char* data, size_t size)
- {
- //static const char* escaped = "\"\\/\b\f\n\r\t";
- struct Parser
- {
- Parser(char* data, size_t /*size*/)
- : data(data)
- {
- }
-
- bool consume(char c)
- {
- if (crow_json_unlikely(*data != c))
- return false;
- data++;
- return true;
- }
-
- void ws_skip()
- {
- while(*data == ' ' || *data == '\t' || *data == '\r' || *data == '\n') ++data;
- };
-
- rvalue decode_string()
- {
- if (crow_json_unlikely(!consume('"')))
- return {};
- char* start = data;
- uint8_t has_escaping = 0;
- while(1)
- {
- if (crow_json_likely(*data != '"' && *data != '\\' && *data != '\0'))
- {
- data ++;
- }
- else if (*data == '"')
- {
- *data = 0;
- *(start-1) = has_escaping;
- data++;
- return {type::String, start, data-1};
- }
- else if (*data == '\\')
- {
- has_escaping = 1;
- data++;
- switch(*data)
- {
- case 'u':
- {
- auto check = [](char c)
- {
- return
- ('0' <= c && c <= '9') ||
- ('a' <= c && c <= 'f') ||
- ('A' <= c && c <= 'F');
- };
- if (!(check(*(data+1)) &&
- check(*(data+2)) &&
- check(*(data+3)) &&
- check(*(data+4))))
- return {};
- }
- data += 5;
- break;
- case '"':
- case '\\':
- case '/':
- case 'b':
- case 'f':
- case 'n':
- case 'r':
- case 't':
- data ++;
- break;
- default:
- return {};
- }
- }
- else
- return {};
- }
- return {};
- }
-
- rvalue decode_list()
- {
- rvalue ret(type::List);
- if (crow_json_unlikely(!consume('[')))
- {
- ret.set_error();
- return ret;
- }
- ws_skip();
- if (crow_json_unlikely(*data == ']'))
- {
- data++;
- return ret;
- }
-
- while(1)
- {
- auto v = decode_value();
- if (crow_json_unlikely(!v))
- {
- ret.set_error();
- break;
- }
- ws_skip();
- ret.emplace_back(std::move(v));
- if (*data == ']')
- {
- data++;
- break;
- }
- if (crow_json_unlikely(!consume(',')))
- {
- ret.set_error();
- break;
- }
- ws_skip();
- }
- return ret;
- }
-
- rvalue decode_number()
- {
- char* start = data;
-
- enum NumberParsingState
- {
- Minus,
- AfterMinus,
- ZeroFirst,
- Digits,
- DigitsAfterPoints,
- E,
- DigitsAfterE,
- Invalid,
- } state{Minus};
- while(crow_json_likely(state != Invalid))
- {
- switch(*data)
- {
- case '0':
- state = (NumberParsingState)"\2\2\7\3\4\6\6"[state];
- /*if (state == NumberParsingState::Minus || state == NumberParsingState::AfterMinus)
- {
- state = NumberParsingState::ZeroFirst;
- }
- else if (state == NumberParsingState::Digits ||
- state == NumberParsingState::DigitsAfterE ||
- state == NumberParsingState::DigitsAfterPoints)
- {
- // ok; pass
- }
- else if (state == NumberParsingState::E)
- {
- state = NumberParsingState::DigitsAfterE;
- }
- else
- return {};*/
- break;
- case '1': case '2': case '3':
- case '4': case '5': case '6':
- case '7': case '8': case '9':
- state = (NumberParsingState)"\3\3\7\3\4\6\6"[state];
- while(*(data+1) >= '0' && *(data+1) <= '9') data++;
- /*if (state == NumberParsingState::Minus || state == NumberParsingState::AfterMinus)
- {
- state = NumberParsingState::Digits;
- }
- else if (state == NumberParsingState::Digits ||
- state == NumberParsingState::DigitsAfterE ||
- state == NumberParsingState::DigitsAfterPoints)
- {
- // ok; pass
- }
- else if (state == NumberParsingState::E)
- {
- state = NumberParsingState::DigitsAfterE;
- }
- else
- return {};*/
- break;
- case '.':
- state = (NumberParsingState)"\7\7\4\4\7\7\7"[state];
- /*
- if (state == NumberParsingState::Digits || state == NumberParsingState::ZeroFirst)
- {
- state = NumberParsingState::DigitsAfterPoints;
- }
- else
- return {};
- */
- break;
- case '-':
- state = (NumberParsingState)"\1\7\7\7\7\6\7"[state];
- /*if (state == NumberParsingState::Minus)
- {
- state = NumberParsingState::AfterMinus;
- }
- else if (state == NumberParsingState::E)
- {
- state = NumberParsingState::DigitsAfterE;
- }
- else
- return {};*/
- break;
- case '+':
- state = (NumberParsingState)"\7\7\7\7\7\6\7"[state];
- /*if (state == NumberParsingState::E)
- {
- state = NumberParsingState::DigitsAfterE;
- }
- else
- return {};*/
- break;
- case 'e': case 'E':
- state = (NumberParsingState)"\7\7\7\5\5\7\7"[state];
- /*if (state == NumberParsingState::Digits ||
- state == NumberParsingState::DigitsAfterPoints)
- {
- state = NumberParsingState::E;
- }
- else
- return {};*/
- break;
- default:
- if (crow_json_likely(state == NumberParsingState::ZeroFirst ||
- state == NumberParsingState::Digits ||
- state == NumberParsingState::DigitsAfterPoints ||
- state == NumberParsingState::DigitsAfterE))
- return {type::Number, start, data};
- else
- return {};
- }
- data++;
- }
-
- return {};
- }
-
- rvalue decode_value()
- {
- switch(*data)
- {
- case '[':
- return decode_list();
- case '{':
- return decode_object();
- case '"':
- return decode_string();
- case 't':
- if (//e-data >= 4 &&
- data[1] == 'r' &&
- data[2] == 'u' &&
- data[3] == 'e')
- {
- data += 4;
- return {type::True};
- }
- else
- return {};
- case 'f':
- if (//e-data >= 5 &&
- data[1] == 'a' &&
- data[2] == 'l' &&
- data[3] == 's' &&
- data[4] == 'e')
- {
- data += 5;
- return {type::False};
- }
- else
- return {};
- case 'n':
- if (//e-data >= 4 &&
- data[1] == 'u' &&
- data[2] == 'l' &&
- data[3] == 'l')
- {
- data += 4;
- return {type::Null};
- }
- else
- return {};
- //case '1': case '2': case '3':
- //case '4': case '5': case '6':
- //case '7': case '8': case '9':
- //case '0': case '-':
- default:
- return decode_number();
- }
- return {};
- }
-
- rvalue decode_object()
- {
- rvalue ret(type::Object);
- if (crow_json_unlikely(!consume('{')))
- {
- ret.set_error();
- return ret;
- }
-
- ws_skip();
-
- if (crow_json_unlikely(*data == '}'))
- {
- data++;
- return ret;
- }
-
- while(1)
- {
- auto t = decode_string();
- if (crow_json_unlikely(!t))
- {
- ret.set_error();
- break;
- }
-
- ws_skip();
- if (crow_json_unlikely(!consume(':')))
- {
- ret.set_error();
- break;
- }
-
- // TODO caching key to speed up (flyweight?)
- auto key = t.s();
-
- ws_skip();
- auto v = decode_value();
- if (crow_json_unlikely(!v))
- {
- ret.set_error();
- break;
- }
- ws_skip();
-
- v.key_ = std::move(key);
- ret.emplace_back(std::move(v));
- if (crow_json_unlikely(*data == '}'))
- {
- data++;
- break;
- }
- if (crow_json_unlikely(!consume(',')))
- {
- ret.set_error();
- break;
- }
- ws_skip();
- }
- return ret;
- }
-
- rvalue parse()
- {
- ws_skip();
- auto ret = decode_value(); // or decode object?
- ws_skip();
- if (ret && *data != '\0')
- ret.set_error();
- return ret;
- }
-
- char* data;
- };
- return Parser(data, size).parse();
- }
- inline rvalue load(const char* data, size_t size)
- {
- char* s = new char[size+1];
- memcpy(s, data, size);
- s[size] = 0;
- auto ret = load_nocopy_internal(s, size);
- if (ret)
- ret.key_.force(s, size);
- else
- delete[] s;
- return ret;
- }
-
- inline rvalue load(const char* data)
- {
- return load(data, strlen(data));
- }
-
- inline rvalue load(const std::string& str)
- {
- return load(str.data(), str.size());
- }
-
- class wvalue
- {
- friend class crow::mustache::template_t;
- public:
- type t() const { return t_; }
- private:
- type t_{type::Null};
- double d {};
- std::string s;
- std::unique_ptr> l;
- std::unique_ptr> o;
- public:
-
- wvalue() {}
-
- wvalue(const rvalue& r)
- {
- t_ = r.t();
- switch(r.t())
- {
- case type::Null:
- case type::False:
- case type::True:
- return;
- case type::Number:
- d = r.d();
- return;
- case type::String:
- s = r.s();
- return;
- case type::List:
- l = std::unique_ptr>(new std::vector{});
- l->reserve(r.size());
- for(auto it = r.begin(); it != r.end(); ++it)
- l->emplace_back(*it);
- return;
- case type::Object:
- o = std::unique_ptr<
- std::unordered_map
- >(
- new std::unordered_map{});
- for(auto it = r.begin(); it != r.end(); ++it)
- o->emplace(it->key(), *it);
- return;
- }
- }
-
- wvalue(wvalue&& r)
- {
- *this = std::move(r);
- }
-
- wvalue& operator = (wvalue&& r)
- {
- t_ = r.t_;
- d = r.d;
- s = std::move(r.s);
- l = std::move(r.l);
- o = std::move(r.o);
- return *this;
- }
-
- void clear()
- {
- t_ = type::Null;
- l.reset();
- o.reset();
- }
-
- void reset()
- {
- t_ = type::Null;
- l.reset();
- o.reset();
- }
-
- wvalue& operator = (std::nullptr_t)
- {
- reset();
- return *this;
- }
- wvalue& operator = (bool value)
- {
- reset();
- if (value)
- t_ = type::True;
- else
- t_ = type::False;
- return *this;
- }
-
- wvalue& operator = (double value)
- {
- reset();
- t_ = type::Number;
- d = value;
- return *this;
- }
-
- wvalue& operator = (unsigned short value)
- {
- reset();
- t_ = type::Number;
- d = (double)value;
- return *this;
- }
-
- wvalue& operator = (short value)
- {
- reset();
- t_ = type::Number;
- d = (double)value;
- return *this;
- }
-
- wvalue& operator = (long long value)
- {
- reset();
- t_ = type::Number;
- d = (double)value;
- return *this;
- }
-
- wvalue& operator = (long value)
- {
- reset();
- t_ = type::Number;
- d = (double)value;
- return *this;
- }
-
- wvalue& operator = (int value)
- {
- reset();
- t_ = type::Number;
- d = (double)value;
- return *this;
- }
-
- wvalue& operator = (unsigned long long value)
- {
- reset();
- t_ = type::Number;
- d = (double)value;
- return *this;
- }
-
- wvalue& operator = (unsigned long value)
- {
- reset();
- t_ = type::Number;
- d = (double)value;
- return *this;
- }
-
- wvalue& operator = (unsigned int value)
- {
- reset();
- t_ = type::Number;
- d = (double)value;
- return *this;
- }
-
- wvalue& operator=(const char* str)
- {
- reset();
- t_ = type::String;
- s = str;
- return *this;
- }
-
- wvalue& operator=(const std::string& str)
- {
- reset();
- t_ = type::String;
- s = str;
- return *this;
- }
-
- wvalue& operator=(std::vector&& v)
- {
- if (t_ != type::List)
- reset();
- t_ = type::List;
- if (!l)
- l = std::unique_ptr>(new std::vector{});
- l->clear();
- l->resize(v.size());
- size_t idx = 0;
- for(auto& x:v)
- {
- (*l)[idx++] = std::move(x);
- }
- return *this;
- }
-
- template
- wvalue& operator=(const std::vector& v)
- {
- if (t_ != type::List)
- reset();
- t_ = type::List;
- if (!l)
- l = std::unique_ptr>(new std::vector{});
- l->clear();
- l->resize(v.size());
- size_t idx = 0;
- for(auto& x:v)
- {
- (*l)[idx++] = x;
- }
- return *this;
- }
-
- wvalue& operator[](unsigned index)
- {
- if (t_ != type::List)
- reset();
- t_ = type::List;
- if (!l)
- l = std::unique_ptr>(new std::vector{});
- if (l->size() < index+1)
- l->resize(index+1);
- return (*l)[index];
- }
-
- int count(const std::string& str)
- {
- if (t_ != type::Object)
- return 0;
- if (!o)
- return 0;
- return o->count(str);
- }
-
- wvalue& operator[](const std::string& str)
- {
- if (t_ != type::Object)
- reset();
- t_ = type::Object;
- if (!o)
- o = std::unique_ptr<
- std::unordered_map
- >(
- new std::unordered_map{});
- return (*o)[str];
- }
-
- std::vector keys() const
- {
- if (t_ != type::Object)
- return {};
- std::vector result;
- for (auto& kv:*o)
- {
- result.push_back(kv.first);
- }
- return result;
- }
-
- size_t estimate_length() const
- {
- switch(t_)
- {
- case type::Null: return 4;
- case type::False: return 5;
- case type::True: return 4;
- case type::Number: return 30;
- case type::String: return 2+s.size()+s.size()/2;
- case type::List:
- {
- size_t sum{};
- if (l)
- {
- for(auto& x:*l)
- {
- sum += 1;
- sum += x.estimate_length();
- }
- }
- return sum+2;
- }
- case type::Object:
- {
- size_t sum{};
- if (o)
- {
- for(auto& kv:*o)
- {
- sum += 2;
- sum += 2+kv.first.size()+kv.first.size()/2;
- sum += kv.second.estimate_length();
- }
- }
- return sum+2;
- }
- }
- return 1;
- }
-
- friend void dump_internal(const wvalue& v, std::string& out);
- friend std::string dump(const wvalue& v);
- };
-
- inline void dump_string(const std::string& str, std::string& out)
- {
- out.push_back('"');
- escape(str, out);
- out.push_back('"');
- }
- inline void dump_internal(const wvalue& v, std::string& out)
- {
- switch(v.t_)
- {
- case type::Null: out += "null"; break;
- case type::False: out += "false"; break;
- case type::True: out += "true"; break;
- case type::Number:
- {
- char outbuf[128];
-#ifdef _MSC_VER
- sprintf_s(outbuf, 128, "%g", v.d);
-#else
- sprintf(outbuf, "%g", v.d);
-#endif
-
- out += outbuf;
- }
- break;
- case type::String: dump_string(v.s, out); break;
- case type::List:
- {
- out.push_back('[');
- if (v.l)
- {
- bool first = true;
- for(auto& x:*v.l)
- {
- if (!first)
- {
- out.push_back(',');
- }
- first = false;
- dump_internal(x, out);
- }
- }
- out.push_back(']');
- }
- break;
- case type::Object:
- {
- out.push_back('{');
- if (v.o)
- {
- bool first = true;
- for(auto& kv:*v.o)
- {
- if (!first)
- {
- out.push_back(',');
- }
- first = false;
- dump_string(kv.first, out);
- out.push_back(':');
- dump_internal(kv.second, out);
- }
- }
- out.push_back('}');
- }
- break;
- }
- }
-
- inline std::string dump(const wvalue& v)
- {
- std::string ret;
- ret.reserve(v.estimate_length());
- dump_internal(v, ret);
- return ret;
- }
-
- //std::vector dump_ref(wvalue& v)
- //{
- //}
- }
-}
-
-#undef crow_json_likely
-#undef crow_json_unlikely
-
-
-
-#pragma once
-#include
-#include
-#include
-#include
-#include
-
-
-namespace crow
-{
- namespace mustache
- {
- using context = json::wvalue;
-
- template_t load(const std::string& filename);
-
- class invalid_template_exception : public std::exception
- {
- public:
- invalid_template_exception(const std::string& msg)
- : msg("crow::mustache error: " + msg)
- {
- }
- virtual const char* what() const throw()
- {
- return msg.c_str();
- }
- std::string msg;
- };
-
- enum class ActionType
- {
- Ignore,
- Tag,
- UnescapeTag,
- OpenBlock,
- CloseBlock,
- ElseBlock,
- Partial,
- };
-
- struct Action
- {
- int start;
- int end;
- int pos;
- ActionType t;
- Action(ActionType t, int start, int end, int pos = 0)
- : start(start), end(end), pos(pos), t(t)
- {}
- };
-
- class template_t
- {
- public:
- template_t(std::string body)
- : body_(std::move(body))
- {
- // {{ {{# {{/ {{^ {{! {{> {{=
- parse();
- }
-
- private:
- std::string tag_name(const Action& action)
- {
- return body_.substr(action.start, action.end - action.start);
- }
- auto find_context(const std::string& name, const std::vector& stack)->std::pair
- {
- if (name == ".")
- {
- return {true, *stack.back()};
- }
- int dotPosition = name.find(".");
- if (dotPosition == (int)name.npos)
- {
- for(auto it = stack.rbegin(); it != stack.rend(); ++it)
- {
- if ((*it)->t() == json::type::Object)
- {
- if ((*it)->count(name))
- return {true, (**it)[name]};
- }
- }
- }
- else
- {
- std::vector dotPositions;
- dotPositions.push_back(-1);
- while(dotPosition != (int)name.npos)
- {
- dotPositions.push_back(dotPosition);
- dotPosition = name.find(".", dotPosition+1);
- }
- dotPositions.push_back(name.size());
- std::vector names;
- names.reserve(dotPositions.size()-1);
- for(int i = 1; i < (int)dotPositions.size(); i ++)
- names.emplace_back(name.substr(dotPositions[i-1]+1, dotPositions[i]-dotPositions[i-1]-1));
-
- for(auto it = stack.rbegin(); it != stack.rend(); ++it)
- {
- context* view = *it;
- bool found = true;
- for(auto jt = names.begin(); jt != names.end(); ++jt)
- {
- if (view->t() == json::type::Object &&
- view->count(*jt))
- {
- view = &(*view)[*jt];
- }
- else
- {
- found = false;
- break;
- }
- }
- if (found)
- return {true, *view};
- }
-
- }
-
- static json::wvalue empty_str;
- empty_str = "";
- return {false, empty_str};
- }
-
- void escape(const std::string& in, std::string& out)
- {
- out.reserve(out.size() + in.size());
- for(auto it = in.begin(); it != in.end(); ++it)
- {
- switch(*it)
- {
- case '&': out += "&"; break;
- case '<': out += "<"; break;
- case '>': out += ">"; break;
- case '"': out += """; break;
- case '\'': out += "'"; break;
- case '/': out += "/"; break;
- default: out += *it; break;
- }
- }
- }
-
- void render_internal(int actionBegin, int actionEnd, std::vector& stack, std::string& out, int indent)
- {
- int current = actionBegin;
-
- if (indent)
- out.insert(out.size(), indent, ' ');
-
- while(current < actionEnd)
- {
- auto& fragment = fragments_[current];
- auto& action = actions_[current];
- render_fragment(fragment, indent, out);
- switch(action.t)
- {
- case ActionType::Ignore:
- // do nothing
- break;
- case ActionType::Partial:
- {
- std::string partial_name = tag_name(action);
- auto partial_templ = load(partial_name);
- int partial_indent = action.pos;
- partial_templ.render_internal(0, partial_templ.fragments_.size()-1, stack, out, partial_indent?indent+partial_indent:0);
- }
- break;
- case ActionType::UnescapeTag:
- case ActionType::Tag:
- {
- auto optional_ctx = find_context(tag_name(action), stack);
- auto& ctx = optional_ctx.second;
- switch(ctx.t())
- {
- case json::type::Number:
- out += json::dump(ctx);
- break;
- case json::type::String:
- if (action.t == ActionType::Tag)
- escape(ctx.s, out);
- else
- out += ctx.s;
- break;
- default:
- throw std::runtime_error("not implemented tag type" + boost::lexical_cast((int)ctx.t()));
- }
- }
- break;
- case ActionType::ElseBlock:
- {
- static context nullContext;
- auto optional_ctx = find_context(tag_name(action), stack);
- if (!optional_ctx.first)
- {
- stack.emplace_back(&nullContext);
- break;
- }
-
- auto& ctx = optional_ctx.second;
- switch(ctx.t())
- {
- case json::type::List:
- if (ctx.l && !ctx.l->empty())
- current = action.pos;
- else
- stack.emplace_back(&nullContext);
- break;
- case json::type::False:
- case json::type::Null:
- stack.emplace_back(&nullContext);
- break;
- default:
- current = action.pos;
- break;
- }
- break;
- }
- case ActionType::OpenBlock:
- {
- auto optional_ctx = find_context(tag_name(action), stack);
- if (!optional_ctx.first)
- {
- current = action.pos;
- break;
- }
-
- auto& ctx = optional_ctx.second;
- switch(ctx.t())
- {
- case json::type::List:
- if (ctx.l)
- for(auto it = ctx.l->begin(); it != ctx.l->end(); ++it)
- {
- stack.push_back(&*it);
- render_internal(current+1, action.pos, stack, out, indent);
- stack.pop_back();
- }
- current = action.pos;
- break;
- case json::type::Number:
- case json::type::String:
- case json::type::Object:
- case json::type::True:
- stack.push_back(&ctx);
- break;
- case json::type::False:
- case json::type::Null:
- current = action.pos;
- break;
- default:
- throw std::runtime_error("{{#: not implemented context type: " + boost::lexical_cast((int)ctx.t()));
- break;
- }
- break;
- }
- case ActionType::CloseBlock:
- stack.pop_back();
- break;
- default:
- throw std::runtime_error("not implemented " + boost::lexical_cast((int)action.t));
- }
- current++;
- }
- auto& fragment = fragments_[actionEnd];
- render_fragment(fragment, indent, out);
- }
- void render_fragment(const std::pair fragment, int indent, std::string& out)
- {
- if (indent)
- {
- for(int i = fragment.first; i < fragment.second; i ++)
- {
- out += body_[i];
- if (body_[i] == '\n' && i+1 != (int)body_.size())
- out.insert(out.size(), indent, ' ');
- }
- }
- else
- out.insert(out.size(), body_, fragment.first, fragment.second-fragment.first);
- }
- public:
- std::string render()
- {
- context empty_ctx;
- std::vector stack;
- stack.emplace_back(&empty_ctx);
-
- std::string ret;
- render_internal(0, fragments_.size()-1, stack, ret, 0);
- return ret;
- }
- std::string render(context& ctx)
- {
- std::vector stack;
- stack.emplace_back(&ctx);
-
- std::string ret;
- render_internal(0, fragments_.size()-1, stack, ret, 0);
- return ret;
- }
-
- private:
-
- void parse()
- {
- std::string tag_open = "{{";
- std::string tag_close = "}}";
-
- std::vector blockPositions;
-
- size_t current = 0;
- while(1)
- {
- size_t idx = body_.find(tag_open, current);
- if (idx == body_.npos)
- {
- fragments_.emplace_back(current, body_.size());
- actions_.emplace_back(ActionType::Ignore, 0, 0);
- break;
- }
- fragments_.emplace_back(current, idx);
-
- idx += tag_open.size();
- size_t endIdx = body_.find(tag_close, idx);
- if (endIdx == idx)
- {
- throw invalid_template_exception("empty tag is not allowed");
- }
- if (endIdx == body_.npos)
- {
- // error, no matching tag
- throw invalid_template_exception("not matched opening tag");
- }
- current = endIdx + tag_close.size();
- switch(body_[idx])
- {
- case '#':
- idx++;
- while(body_[idx] == ' ') idx++;
- while(body_[endIdx-1] == ' ') endIdx--;
- blockPositions.emplace_back(actions_.size());
- actions_.emplace_back(ActionType::OpenBlock, idx, endIdx);
- break;
- case '/':
- idx++;
- while(body_[idx] == ' ') idx++;
- while(body_[endIdx-1] == ' ') endIdx--;
- {
- auto& matched = actions_[blockPositions.back()];
- if (body_.compare(idx, endIdx-idx,
- body_, matched.start, matched.end - matched.start) != 0)
- {
- throw invalid_template_exception("not matched {{# {{/ pair: " +
- body_.substr(matched.start, matched.end - matched.start) + ", " +
- body_.substr(idx, endIdx-idx));
- }
- matched.pos = actions_.size();
- }
- actions_.emplace_back(ActionType::CloseBlock, idx, endIdx, blockPositions.back());
- blockPositions.pop_back();
- break;
- case '^':
- idx++;
- while(body_[idx] == ' ') idx++;
- while(body_[endIdx-1] == ' ') endIdx--;
- blockPositions.emplace_back(actions_.size());
- actions_.emplace_back(ActionType::ElseBlock, idx, endIdx);
- break;
- case '!':
- // do nothing action
- actions_.emplace_back(ActionType::Ignore, idx+1, endIdx);
- break;
- case '>': // partial
- idx++;
- while(body_[idx] == ' ') idx++;
- while(body_[endIdx-1] == ' ') endIdx--;
- actions_.emplace_back(ActionType::Partial, idx, endIdx);
- break;
- case '{':
- if (tag_open != "{{" || tag_close != "}}")
- throw invalid_template_exception("cannot use triple mustache when delimiter changed");
-
- idx ++;
- if (body_[endIdx+2] != '}')
- {
- throw invalid_template_exception("{{{: }}} not matched");
- }
- while(body_[idx] == ' ') idx++;
- while(body_[endIdx-1] == ' ') endIdx--;
- actions_.emplace_back(ActionType::UnescapeTag, idx, endIdx);
- current++;
- break;
- case '&':
- idx ++;
- while(body_[idx] == ' ') idx++;
- while(body_[endIdx-1] == ' ') endIdx--;
- actions_.emplace_back(ActionType::UnescapeTag, idx, endIdx);
- break;
- case '=':
- // tag itself is no-op
- idx ++;
- actions_.emplace_back(ActionType::Ignore, idx, endIdx);
- endIdx --;
- if (body_[endIdx] != '=')
- throw invalid_template_exception("{{=: not matching = tag: "+body_.substr(idx, endIdx-idx));
- endIdx --;
- while(body_[idx] == ' ') idx++;
- while(body_[endIdx] == ' ') endIdx--;
- endIdx++;
- {
- bool succeeded = false;
- for(size_t i = idx; i < endIdx; i++)
- {
- if (body_[i] == ' ')
- {
- tag_open = body_.substr(idx, i-idx);
- while(body_[i] == ' ') i++;
- tag_close = body_.substr(i, endIdx-i);
- if (tag_open.empty())
- throw invalid_template_exception("{{=: empty open tag");
- if (tag_close.empty())
- throw invalid_template_exception("{{=: empty close tag");
-
- if (tag_close.find(" ") != tag_close.npos)
- throw invalid_template_exception("{{=: invalid open/close tag: "+tag_open+" " + tag_close);
- succeeded = true;
- break;
- }
- }
- if (!succeeded)
- throw invalid_template_exception("{{=: cannot find space between new open/close tags");
- }
- break;
- default:
- // normal tag case;
- while(body_[idx] == ' ') idx++;
- while(body_[endIdx-1] == ' ') endIdx--;
- actions_.emplace_back(ActionType::Tag, idx, endIdx);
- break;
- }
- }
-
- // removing standalones
- for(int i = actions_.size()-2; i >= 0; i --)
- {
- if (actions_[i].t == ActionType::Tag || actions_[i].t == ActionType::UnescapeTag)
- continue;
- auto& fragment_before = fragments_[i];
- auto& fragment_after = fragments_[i+1];
- bool is_last_action = i == (int)actions_.size()-2;
- bool all_space_before = true;
- int j, k;
- for(j = fragment_before.second-1;j >= fragment_before.first;j--)
- {
- if (body_[j] != ' ')
- {
- all_space_before = false;
- break;
- }
- }
- if (all_space_before && i > 0)
- continue;
- if (!all_space_before && body_[j] != '\n')
- continue;
- bool all_space_after = true;
- for(k = fragment_after.first; k < (int)body_.size() && k < fragment_after.second; k ++)
- {
- if (body_[k] != ' ')
- {
- all_space_after = false;
- break;
- }
- }
- if (all_space_after && !is_last_action)
- continue;
- if (!all_space_after &&
- !(
- body_[k] == '\n'
- ||
- (body_[k] == '\r' &&
- k + 1 < (int)body_.size() &&
- body_[k+1] == '\n')))
- continue;
- if (actions_[i].t == ActionType::Partial)
- {
- actions_[i].pos = fragment_before.second - j - 1;
- }
- fragment_before.second = j+1;
- if (!all_space_after)
- {
- if (body_[k] == '\n')
- k++;
- else
- k += 2;
- fragment_after.first = k;
- }
- }
- }
-
- std::vector> fragments_;
- std::vector actions_;
- std::string body_;
- };
-
- inline template_t compile(const std::string& body)
- {
- return template_t(body);
- }
- namespace detail
- {
- inline std::string& get_template_base_directory_ref()
- {
- static std::string template_base_directory = "templates";
- return template_base_directory;
- }
- }
-
- inline std::string default_loader(const std::string& filename)
- {
- std::string path = detail::get_template_base_directory_ref();
- if (!(path.back() == '/' || path.back() == '\\'))
- path += '/';
- path += filename;
- std::ifstream inf(path);
- if (!inf)
- return {};
- return {std::istreambuf_iterator(inf), std::istreambuf_iterator()};
- }
-
- namespace detail
- {
- inline std::function& get_loader_ref()
- {
- static std::function loader = default_loader;
- return loader;
- }
- }
-
- inline void set_base(const std::string& path)
- {
- auto& base = detail::get_template_base_directory_ref();
- base = path;
- if (base.back() != '\\' &&
- base.back() != '/')
- {
- base += '/';
- }
- }
-
- inline void set_loader(std::function loader)
- {
- detail::get_loader_ref() = std::move(loader);
- }
-
- inline std::string load_text(const std::string& filename)
- {
- return detail::get_loader_ref()(filename);
- }
-
- inline template_t load(const std::string& filename)
- {
- return compile(detail::get_loader_ref()(filename));
- }
- }
-}
-
-
-
-#pragma once
-
-#include
-#include
-#include
-#include
-#include
-#include
-
-
-
-
-namespace crow
-{
- enum class LogLevel
- {
-#ifndef ERROR
- DEBUG = 0,
- INFO,
- WARNING,
- ERROR,
- CRITICAL,
-#endif
-
- Debug = 0,
- Info,
- Warning,
- Error,
- Critical,
- };
-
- class ILogHandler {
- public:
- virtual void log(std::string message, LogLevel level) = 0;
- };
-
- class CerrLogHandler : public ILogHandler {
- public:
- void log(std::string message, LogLevel /*level*/) override {
- std::cerr << message;
- }
- };
-
- class logger {
-
- private:
- //
- static std::string timestamp()
- {
- char date[32];
- time_t t = time(0);
-
- tm my_tm;
-
-#ifdef _MSC_VER
- gmtime_s(&my_tm, &t);
-#else
- gmtime_r(&t, &my_tm);
-#endif
-
- size_t sz = strftime(date, sizeof(date), "%Y-%m-%d %H:%M:%S", &my_tm);
- return std::string(date, date+sz);
- }
-
- public:
-
-
- logger(std::string prefix, LogLevel level) : level_(level) {
- #ifdef CROW_ENABLE_LOGGING
- stringstream_ << "(" << timestamp() << ") [" << prefix << "] ";
- #endif
-
- }
- ~logger() {
- #ifdef CROW_ENABLE_LOGGING
- if(level_ >= get_current_log_level()) {
- stringstream_ << std::endl;
- get_handler_ref()->log(stringstream_.str(), level_);
- }
- #endif
- }
-
- //
- template
- logger& operator<<(T const &value) {
-
- #ifdef CROW_ENABLE_LOGGING
- if(level_ >= get_current_log_level()) {
- stringstream_ << value;
- }
- #endif
- return *this;
- }
-
- //
- static void setLogLevel(LogLevel level) {
- get_log_level_ref() = level;
- }
-
- static void setHandler(ILogHandler* handler) {
- get_handler_ref() = handler;
- }
-
- static LogLevel get_current_log_level() {
- return get_log_level_ref();
- }
-
- private:
- //
- static LogLevel& get_log_level_ref()
- {
- static LogLevel current_level = (LogLevel)CROW_LOG_LEVEL;
- return current_level;
- }
- static ILogHandler*& get_handler_ref()
- {
- static CerrLogHandler default_handler;
- static ILogHandler* current_handler = &default_handler;
- return current_handler;
- }
-
- //
- std::ostringstream stringstream_;
- LogLevel level_;
- };
-}
-
-#define CROW_LOG_CRITICAL \
- if (crow::logger::get_current_log_level() <= crow::LogLevel::Critical) \
- crow::logger("CRITICAL", crow::LogLevel::Critical)
-#define CROW_LOG_ERROR \
- if (crow::logger::get_current_log_level() <= crow::LogLevel::Error) \
- crow::logger("ERROR ", crow::LogLevel::Error)
-#define CROW_LOG_WARNING \
- if (crow::logger::get_current_log_level() <= crow::LogLevel::Warning) \
- crow::logger("WARNING ", crow::LogLevel::Warning)
-#define CROW_LOG_INFO \
- if (crow::logger::get_current_log_level() <= crow::LogLevel::Info) \
- crow::logger("INFO ", crow::LogLevel::Info)
-#define CROW_LOG_DEBUG \
- if (crow::logger::get_current_log_level() <= crow::LogLevel::Debug) \
- crow::logger("DEBUG ", crow::LogLevel::Debug)
-
-
-
-
-#pragma once
-
-#include
-#include
-#include
-#include
-#include
-
-
-
-
-namespace crow
-{
- namespace detail
- {
- // fast timer queue for fixed tick value.
- class dumb_timer_queue
- {
- public:
- using key = std::pair;
-
- void cancel(key& k)
- {
- auto self = k.first;
- k.first = nullptr;
- if (!self)
- return;
-
- unsigned int index = (unsigned int)(k.second - self->step_);
- if (index < self->dq_.size())
- self->dq_[index].second = nullptr;
- }
-
- key add(std::function f)
- {
- dq_.emplace_back(std::chrono::steady_clock::now(), std::move(f));
- int ret = step_+dq_.size()-1;
-
- CROW_LOG_DEBUG << "timer add inside: " << this << ' ' << ret ;
- return {this, ret};
- }
-
- void process()
- {
- if (!io_service_)
- return;
-
- auto now = std::chrono::steady_clock::now();
- while(!dq_.empty())
- {
- auto& x = dq_.front();
- if (now - x.first < std::chrono::seconds(tick))
- break;
- if (x.second)
- {
- CROW_LOG_DEBUG << "timer call: " << this << ' ' << step_;
- // we know that timer handlers are very simple currenty; call here
- x.second();
- }
- dq_.pop_front();
- step_++;
- }
- }
-
- void set_io_service(boost::asio::io_service& io_service)
- {
- io_service_ = &io_service;
- }
-
- dumb_timer_queue() noexcept
- {
- }
-
- private:
-
- int tick{5};
- boost::asio::io_service* io_service_{};
- std::deque>> dq_;
- int step_{};
- };
- }
-}
-
-
-
-#pragma once
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-
-
-
-
-namespace crow
-{
- namespace black_magic
- {
-#ifndef CROW_MSVC_WORKAROUND
- struct OutOfRange
- {
- OutOfRange(unsigned /*pos*/, unsigned /*length*/) {}
- };
- constexpr unsigned requires_in_range( unsigned i, unsigned len )
- {
- return i >= len ? throw OutOfRange(i, len) : i;
- }
-
- class const_str
- {
- const char * const begin_;
- unsigned size_;
-
- public:
- template< unsigned N >
- constexpr const_str( const char(&arr)[N] ) : begin_(arr), size_(N - 1) {
- static_assert( N >= 1, "not a string literal");
- }
- constexpr char operator[]( unsigned i ) const {
- return requires_in_range(i, size_), begin_[i];
- }
-
- constexpr operator const char *() const {
- return begin_;
- }
-
- constexpr const char* begin() const { return begin_; }
- constexpr const char* end() const { return begin_ + size_; }
-
- constexpr unsigned size() const {
- return size_;
- }
- };
-
- constexpr unsigned find_closing_tag(const_str s, unsigned p)
- {
- return s[p] == '>' ? p : find_closing_tag(s, p+1);
- }
-
- constexpr bool is_valid(const_str s, unsigned i = 0, int f = 0)
- {
- return
- i == s.size()
- ? f == 0 :
- f < 0 || f >= 2
- ? false :
- s[i] == '<'
- ? is_valid(s, i+1, f+1) :
- s[i] == '>'
- ? is_valid(s, i+1, f-1) :
- is_valid(s, i+1, f);
- }
-
- constexpr bool is_equ_p(const char* a, const char* b, unsigned n)
- {
- return
- *a == 0 && *b == 0 && n == 0
- ? true :
- (*a == 0 || *b == 0)
- ? false :
- n == 0
- ? true :
- *a != *b
- ? false :
- is_equ_p(a+1, b+1, n-1);
- }
-
- constexpr bool is_equ_n(const_str a, unsigned ai, const_str b, unsigned bi, unsigned n)
- {
- return
- ai + n > a.size() || bi + n > b.size()
- ? false :
- n == 0
- ? true :
- a[ai] != b[bi]
- ? false :
- is_equ_n(a,ai+1,b,bi+1,n-1);
- }
-
- constexpr bool is_int(const_str s, unsigned i)
- {
- return is_equ_n(s, i, "", 0, 5);
- }
-
- constexpr bool is_uint(const_str s, unsigned i)
- {
- return is_equ_n(s, i, "", 0, 6);
- }
-
- constexpr bool is_float(const_str s, unsigned i)
- {
- return is_equ_n(s, i, "", 0, 7) ||
- is_equ_n(s, i, "", 0, 8);
- }
-
- constexpr bool is_str(const_str s, unsigned i)
- {
- return is_equ_n(s, i, "", 0, 5) ||
- is_equ_n(s, i, "", 0, 8);
- }
-
- constexpr bool is_path(const_str s, unsigned i)
- {
- return is_equ_n(s, i, "", 0, 6);
- }
-#endif
- template
- struct parameter_tag
- {
- static const int value = 0;
- };
-#define CROW_INTERNAL_PARAMETER_TAG(t, i) \
-template <> \
-struct parameter_tag \
-{ \
- static const int value = i; \
-}
- CROW_INTERNAL_PARAMETER_TAG(int, 1);
- CROW_INTERNAL_PARAMETER_TAG(char, 1);
- CROW_INTERNAL_PARAMETER_TAG(short, 1);
- CROW_INTERNAL_PARAMETER_TAG(long, 1);
- CROW_INTERNAL_PARAMETER_TAG(long long, 1);
- CROW_INTERNAL_PARAMETER_TAG(unsigned int, 2);
- CROW_INTERNAL_PARAMETER_TAG(unsigned char, 2);
- CROW_INTERNAL_PARAMETER_TAG(unsigned short, 2);
- CROW_INTERNAL_PARAMETER_TAG(unsigned long, 2);
- CROW_INTERNAL_PARAMETER_TAG(unsigned long long, 2);
- CROW_INTERNAL_PARAMETER_TAG(double, 3);
- CROW_INTERNAL_PARAMETER_TAG(std::string, 4);
-#undef CROW_INTERNAL_PARAMETER_TAG
- template
- struct compute_parameter_tag_from_args_list;
-
- template <>
- struct compute_parameter_tag_from_args_list<>
- {
- static const int value = 0;
- };
-
- template
- struct compute_parameter_tag_from_args_list
- {
- static const int sub_value =
- compute_parameter_tag_from_args_list::value;
- static const int value =
- parameter_tag::type>::value
- ? sub_value* 6 + parameter_tag::type>::value
- : sub_value;
- };
-
- static inline bool is_parameter_tag_compatible(uint64_t a, uint64_t b)
- {
- if (a == 0)
- return b == 0;
- if (b == 0)
- return a == 0;
- int sa = a%6;
- int sb = a%6;
- if (sa == 5) sa = 4;
- if (sb == 5) sb = 4;
- if (sa != sb)
- return false;
- return is_parameter_tag_compatible(a/6, b/6);
- }
-
- static inline unsigned find_closing_tag_runtime(const char* s, unsigned p)
- {
- return
- s[p] == 0
- ? throw std::runtime_error("unmatched tag <") :
- s[p] == '>'
- ? p : find_closing_tag_runtime(s, p + 1);
- }
-
- static inline uint64_t get_parameter_tag_runtime(const char* s, unsigned p = 0)
- {
- return
- s[p] == 0
- ? 0 :
- s[p] == '<' ? (
- std::strncmp(s+p, "", 5) == 0
- ? get_parameter_tag_runtime(s, find_closing_tag_runtime(s, p)) * 6 + 1 :
- std::strncmp(s+p, "", 6) == 0
- ? get_parameter_tag_runtime(s, find_closing_tag_runtime(s, p)) * 6 + 2 :
- (std::strncmp(s+p, "", 7) == 0 ||
- std::strncmp(s+p, "", 8) == 0)
- ? get_parameter_tag_runtime(s, find_closing_tag_runtime(s, p)) * 6 + 3 :
- (std::strncmp(s+p, "", 5) == 0 ||
- std::strncmp(s+p, "", 8) == 0)
- ? get_parameter_tag_runtime(s, find_closing_tag_runtime(s, p)) * 6 + 4 :
- std::strncmp(s+p, "", 6) == 0
- ? get_parameter_tag_runtime(s, find_closing_tag_runtime(s, p)) * 6 + 5 :
- throw std::runtime_error("invalid parameter type")
- ) :
- get_parameter_tag_runtime(s, p+1);
- }
-#ifndef CROW_MSVC_WORKAROUND
- constexpr uint64_t get_parameter_tag(const_str s, unsigned p = 0)
- {
- return
- p == s.size()
- ? 0 :
- s[p] == '<' ? (
- is_int(s, p)
- ? get_parameter_tag(s, find_closing_tag(s, p)) * 6 + 1 :
- is_uint(s, p)
- ? get_parameter_tag(s, find_closing_tag(s, p)) * 6 + 2 :
- is_float(s, p)
- ? get_parameter_tag(s, find_closing_tag(s, p)) * 6 + 3 :
- is_str(s, p)
- ? get_parameter_tag(s, find_closing_tag(s, p)) * 6 + 4 :
- is_path(s, p)
- ? get_parameter_tag(s, find_closing_tag(s, p)) * 6 + 5 :
- throw std::runtime_error("invalid parameter type")
- ) :
- get_parameter_tag(s, p+1);
- }
-#endif
-
- template
- struct S
- {
- template
- using push = S;
- template
- using push_back = S;
- template class U>
- using rebind = U;
- };
-template
- struct CallHelper;
- template
- struct CallHelper>
- {
- template ()(std::declval()...))
- >
- static char __test(int);
-
- template
- static int __test(...);
-
- static constexpr bool value = sizeof(__test(0)) == sizeof(char);
- };
-
-
- template
- struct single_tag_to_type
- {
- };
-
- template <>
- struct single_tag_to_type<1>
- {
- using type = int64_t;
- };
-
- template <>
- struct single_tag_to_type<2>
- {
- using type = uint64_t;
- };
-
- template <>
- struct single_tag_to_type<3>
- {
- using type = double;
- };
-
- template <>
- struct single_tag_to_type<4>
- {
- using type = std::string;
- };
-
- template <>
- struct single_tag_to_type<5>
- {
- using type = std::string;
- };
-
-
- template
- struct arguments
- {
- using subarguments = typename arguments::type;
- using type =
- typename subarguments::template push::type>;
- };
-
- template <>
- struct arguments<0>
- {
- using type = S<>;
- };
-
- template
- struct last_element_type
- {
- using type = typename std::tuple_element>::type;
- };
-
-
- template <>
- struct last_element_type<>
- {
- };
-
-
- // from http://stackoverflow.com/questions/13072359/c11-compile-time-array-with-logarithmic-evaluation-depth
- template using Invoke = typename T::type;
-
- template struct seq{ using type = seq; };
-
- template struct concat;
-
- template
- struct concat, seq>
- : seq{};
-
- template
- using Concat = Invoke>;
-
- template struct gen_seq;
- template using GenSeq = Invoke>;
-
- template
- struct gen_seq : Concat, GenSeq>{};
-
- template<> struct gen_seq<0> : seq<>{};
- template<> struct gen_seq<1> : seq<0>{};
-
- template
- struct pop_back_helper;
-
- template
- struct pop_back_helper, Tuple>
- {
- template class U>
- using rebind = U::type...>;
- };
-
- template
- struct pop_back //: public pop_back_helper::type, std::tuple>
- {
- template class U>
- using rebind = typename pop_back_helper::type, std::tuple>::template rebind;
- };
-
- template <>
- struct pop_back<>
- {
- template class U>
- using rebind = U<>;
- };
-
- // from http://stackoverflow.com/questions/2118541/check-if-c0x-parameter-pack-contains-a-type
- template < typename Tp, typename... List >
- struct contains : std::true_type {};
-
- template < typename Tp, typename Head, typename... Rest >
- struct contains
- : std::conditional< std::is_same::value,
- std::true_type,
- contains
- >::type {};
-
- template < typename Tp >
- struct contains : std::false_type {};
-
- template
- struct empty_context
- {
- };
-
- template
- struct promote
- {
- using type = T;
- };
-
-#define CROW_INTERNAL_PROMOTE_TYPE(t1, t2) \
- template<> \
- struct promote \
- { \
- using type = t2; \
- }
-
- CROW_INTERNAL_PROMOTE_TYPE(char, int64_t);
- CROW_INTERNAL_PROMOTE_TYPE(short, int64_t);
- CROW_INTERNAL_PROMOTE_TYPE(int, int64_t);
- CROW_INTERNAL_PROMOTE_TYPE(long, int64_t);
- CROW_INTERNAL_PROMOTE_TYPE(long long, int64_t);
- CROW_INTERNAL_PROMOTE_TYPE(unsigned char, uint64_t);
- CROW_INTERNAL_PROMOTE_TYPE(unsigned short, uint64_t);
- CROW_INTERNAL_PROMOTE_TYPE(unsigned int, uint64_t);
- CROW_INTERNAL_PROMOTE_TYPE(unsigned long, uint64_t);
- CROW_INTERNAL_PROMOTE_TYPE(unsigned long long, uint64_t);
- CROW_INTERNAL_PROMOTE_TYPE(float, double);
-#undef CROW_INTERNAL_PROMOTE_TYPE
-
- template