diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..378c4749 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: +- package-ecosystem: github-actions + directory: "/" + schedule: + interval: "daily" + target-branch: "main" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..717871f2 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,140 @@ +name: CI +permissions: read-all + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + + +# When a PR is updated, cancel the jobs from the previous version. Merges +# do not define head_ref, so use run_id to never cancel those jobs. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + TinyCBOR: + timeout-minutes: 45 + # Common environment variables + env: + HOMEBREW_NO_INSTALL_CLEANUP: 1 + HOMEBREW_NO_ANALYTICS: 1 + + strategy: + # Always run all jobs in the matrix, even if one fails. + fail-fast: false + matrix: + os: [ ubuntu-latest ] + build_cfg: [ + { "name": "gcc-no-math", + "flags": + '{ "QMAKESPEC": "linux-gcc-no-math", + "EVAL": "export CXX=false && touch math.h float.h", + "CFLAGS": "-ffreestanding -DCBOR_NO_FLOATING_POINT -Os", + "LDFLAGS": "-Wl,--no-undefined", + "LDLIBS": "" + }', + }, + { "name": "gcc-freestanding", + "flags": + '{ "QMAKESPEC": "linux-gcc-freestanding", + "EVAL": "export CXX=false", + "CFLAGS": "-ffreestanding -Os", + "LDFLAGS": "-Wl,--no-undefined -lm" + }', + }, + { "name": "clang", + "flags": + '{ "QMAKESPEC": "linux-clang", + "EVAL": "export CC=clang && export CXX=clang++", + "CFLAGS": "-Oz", + "LDFLAGS": "-Wl,--no-undefined -lm", + "QMAKEFLAGS": "-config release", + "MAKEFLAGS": "-s", + "TESTARGS": "-silent" + }', + }, + { "name": "linux-g++", + "flags": + '{ "QMAKESPEC": "linux-g++", + "EVAL": "export CC=gcc && export CXX=g++", + "CFLAGS": "-Os", + "LDFLAGS": "-Wl,--no-undefined -lm", + "QMAKEFLAGS": "-config release", + "QT_NO_CPU_FEATURE": "rdrnd" + }' + } + ] + include: + - os: macos-13 + build_cfg: { "name": "clang", + "flags": + '{ "QMAKESPEC": "macx-clang", + "EVAL": "export CC=clang && export CXX=clang++", + "CFLAGS": "-Oz", + "QMAKEFLAGS": "-config debug", + "MAKEFLAGS": "-s", + "TESTARGS": "-silent", + "PATH": "/usr/local/opt/qt/bin:$PATH" + }' + } + + # Default job name is too long to be visible in the "Checks" tab. + name: ${{ matrix.os }}/${{ matrix.build_cfg.name }} + # The type of runner that the job will run on + runs-on: ${{ matrix.os }} + steps: + - name: Clone tinycbor + uses: actions/checkout@v4 + + - name: install Linux software + if: matrix.os == 'ubuntu-latest' + run: | + # Need a recent Valgrind, otherwise debug info cannot be read. + sudo snap install valgrind --classic + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + doxygen \ + jq \ + libc6-dbg \ + libcjson-dev \ + libfuntools-dev \ + qtbase5-dev + + - name: install macOS software + if: runner.os == 'macOS' + run: | + # Doxygen 1.9.7 is broken with ifdefs again, install 1.9.4 which works. + wget https://raw.githubusercontent.com/Homebrew/homebrew-core/41828ee36b96e35b63b2a4c8cfc2df2c3728944a/Formula/doxygen.rb + brew install doxygen.rb + rm doxygen.rb + brew install qt cjson + + - name: Execute tests + run: | + set -x + PATH=`echo /opt/qt*/bin`:$PATH + eval $(echo '${{ matrix.build_cfg.flags }}' | jq -r 'to_entries[] | "\(.key)=\"\(.value)\""') + eval "$EVAL" + # FIXME: remove -Wno-error-line below. + export CFLAGS="$CFLAGS -Wno-error=implicit-function-declaration" + make OUT=.config V=1 -s -f Makefile.configure configure && cat .config + make -k \ + CFLAGS="$CFLAGS -march=native -g1 -Wall -Wextra -Werror" \ + CPPFLAGS="-DNDEBUG -DCBOR_ENCODER_WRITER_CONTROL=-1 -DCBOR_PARSER_READER_CONTROL=-1" \ + lib/libtinycbor.a + size lib/libtinycbor.a | tee sizes + make -s clean + make -k \ + CFLAGS="$CFLAGS -O0 -g" \ + LDFLAGS="$LDFLAGS" ${LDLIBS+LDLIBS="$LDLIBS"} + grep -q freestanding-pass .config || make \ + QMAKEFLAGS="$QMAKEFLAGS QMAKE_CXX=$CXX" \ + tests/Makefile + grep -q freestanding-pass .config || \ + (cd tests && make TESTARGS=-silent check -k \ + TESTRUNNER=`which valgrind 2>/dev/null`) + make -s clean + ! [ $BUILD_DOCS ] || ./scripts/update-docs.sh diff --git a/Makefile b/Makefile index 26925eee..6df16553 100644 --- a/Makefile +++ b/Makefile @@ -65,20 +65,6 @@ VERSION = $(shell cat $(SRCDIR)VERSION) SOVERSION = $(shell cut -f1-2 -d. $(SRCDIR)VERSION) PACKAGE = tinycbor-$(VERSION) -# Check that QMAKE is Qt 5 -ifeq ($(origin QMAKE),file) - check_qmake = $(strip $(shell $(1) -query QT_VERSION 2>/dev/null | cut -b1)) - ifneq ($(call check_qmake,$(QMAKE)),5) - QMAKE := qmake -qt5 - ifneq ($(call check_qmake,$(QMAKE)),5) - QMAKE := qmake-qt5 - ifneq ($(call check_qmake,$(QMAKE)),5) - QMAKE := @echo >&2 $(MAKEFILE): Cannot find a Qt 5 qmake; false - endif - endif - endif -endif - -include .config ifeq ($(wildcard .config),) diff --git a/Makefile.configure b/Makefile.configure index 16bab6bb..d906ef1b 100644 --- a/Makefile.configure +++ b/Makefile.configure @@ -15,11 +15,12 @@ PROGRAM-freestanding += int main() {} CCFLAGS-freestanding = $(CFLAGS) PROGRAM-cjson = \#include \n +PROGRAM-cjson += \#include \n PROGRAM-cjson += \#include \n -PROGRAM-cjson += int main() { return cJSON_False; } -CCFLAGS-cjson = -I$(dir $(MAKEFILE))src +PROGRAM-cjson += int main() { double d = NAN; return cJSON_False; } +CCFLAGS-cjson = -I. -I$(dir $(MAKEFILE))src PROGRAM-system-cjson = $(PROGRAM-cjson) -CCFLAGS-system-cjson = -lcjson +CCFLAGS-system-cjson = -I. -lcjson sink: @echo >&2 Please run from the top-level Makefile. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..373608b6 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,5 @@ +# Security Policy +Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation. + +## Reporting a Vulnerability +Please report any security vulnerabilities in this project utilizing the guidelines [here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html). diff --git a/examples/bufferedreader.c b/examples/bufferedreader.c new file mode 100644 index 00000000..a4a87a4a --- /dev/null +++ b/examples/bufferedreader.c @@ -0,0 +1,783 @@ +/* vim: set sw=4 ts=4 et tw=78: */ + +/** + * \brief An example of a buffered CBOR file reader using low-level + * POSIX file I/O as might be implemented in a microcontroller + * RTOS. + * + * \author Stuart Longland + * + * \copyright tinycbor project contributors + * + * \file bufferedwriter.c + */ + +/* Includes for POSIX low-level file I/O */ +#include +#include +#include +#include + +/* Sanity check routine */ +#include + +/* Pull in "standard" integer types */ +#include + +/* Pull in definitions for printf and errno */ +#include +#include +#include + +/* For example usage */ +#include + +#include "../src/cbor.h" + +/** + * Context for the file reader. This stores the file descriptor, a pointer to + * the read buffer, and context pointers. The assumption here is that the + * CBOR document being read is less than 64KiB (65536 bytes) in size. + */ +struct filereader +{ + /** + * Read buffer. This must be allocated by the caller, and sized + * appropriately since the buffer must be big enough to accommodate + * entire string chunks embedded in the CBOR document. + */ + uint8_t* buffer; + + /** + * File descriptor, returned by the `open` system call. + */ + int fd; + + /** + * Size of the file in bytes. + */ + uint16_t file_sz; + + /** + * Size of the read buffer in bytes. + */ + uint16_t buffer_sz; + + /** + * Read position within the file. This basically describes where + * `buffer[0]` came from in the source file. + */ + uint16_t pos; + + /** + * Number of bytes stored in the buffer presently. + */ + uint16_t used_sz; + + /** + * Block size. When reading from the file, we round up to whole multiples + * of this block size to improve I/O efficiency. + */ + uint16_t block_sz; +}; + +/* Implementation routines */ + +/** + * Return the nearest (earlier) position that is on a block boundary. + */ +static uint16_t filereader_get_block_pos( + const struct filereader * const context, + uint16_t pos +) { + return context->block_sz * (pos / context->block_sz); +} + +/** + * Retrieve a pointer to the region defined by \a pos and \a sz. + * + * \retval NULL The region is not contained in the buffer. + */ +static uint8_t *filereader_get_ptr( + struct filereader *context, uint16_t pos, uint16_t sz +) { + /* Sanity check, disallow `sz` bigger than buffer content */ + if (sz > context->used_sz) { + return NULL; + } + + /* Is `pos` off the start of the buffer? */ + if (pos < context->pos) { + return NULL; + } + + /* Is `pos + sz` off the end of the buffer? */ + if ((pos + sz) > (context->pos + context->used_sz)) { + return NULL; + } + + /* We should be good */ + return &(context->buffer[pos - context->pos]); +} + +/** + * Copy the data from the requested position in the file to the buffer. + */ +static int filereader_read( + struct filereader *context, uint16_t pos, uint16_t sz, uint8_t *wptr +) { + /* Seek to the required file position */ + off_t seek_res = lseek(context->fd, pos, SEEK_SET); + if (seek_res != pos) { + /* We failed */ + return -errno; + } + + /* Perform the read */ + ssize_t read_res = read(context->fd, wptr, sz); + if (read_res != sz) { + /* Truncated read */ + return -errno; + } + + return sz; +} + +/** + * Prepend \arg sz bytes from the file to the buffer, shifting the + * read window back \arg sz bytes. + * + * \param context Reader context + * \param sz Number of bytes to read + * + * \retval ≥0 Number of bytes read into the buffer. + * \retval <0 Read failure, value is `errno` negated. + */ +static int filereader_prepend_buffer(struct filereader *context, uint16_t sz) { + /* Compute read position */ + const uint16_t pos = context->pos - sz; + + /* Shuffle existing data forward by sz bytes */ + memmove( + &(context->buffer[sz]), context->buffer, + context->buffer_sz - sz + ); + + /* Copy the data in */ + int read_res = filereader_read(context, pos, sz, context->buffer); + if (read_res >= 0) { + context->pos = pos; + if ((context->used_sz + sz) < context->buffer_sz) { + context->used_sz += sz; + } + } + + return read_res; +} + +/** + * Append \arg sz bytes from the file to the buffer, shifting the + * read window forward \arg sz bytes. + * + * \param context Reader context + * \param sz Number of bytes to read + * + * \retval ≥0 Number of bytes read into the buffer. + * \retval <0 Read failure, value is `errno` negated. + */ +static int filereader_append_buffer(struct filereader *context, uint16_t sz) { + /* Compute read position */ + const uint16_t pos = context->pos + context->used_sz; + + /* Is there room? */ + if ((context->buffer_sz - context->used_sz) < sz) { + /* Shuffle existing data forward by sz bytes */ + memmove( + context->buffer, &(context->buffer[sz]), + context->buffer_sz - sz + ); + context->pos += sz; + context->used_sz -= sz; + } + + /* Copy the data in */ + int read_res = filereader_read( + context, pos, sz, + &(context->buffer[context->used_sz]) + ); + if (read_res >= 0) { + context->used_sz += sz; + } + + return read_res; +} + +/** + * Read data from the file and place it in the buffer, shuffling + * existing data around as required. + * + * \param context Reader context + * \param pos Position in the file to start reading + * \param sz Number of bytes to read + * + * \retval ≥0 Number of bytes read into the buffer. + * \retval <0 Read failure, value is `errno` negated. + */ +static int filereader_load_buffer( + struct filereader *context, uint16_t pos, uint16_t sz +) { + /* Compute the end position (not-inclusive) */ + uint16_t end = pos + sz; + + /* Is this in the buffer already? */ + if ( + (pos < context->pos) + || (end > (context->pos + context->used_sz)) + ) { + /* Make a note of the current buffer state */ + uint16_t buffer_end = context->pos + context->used_sz; + uint16_t buffer_rem = context->buffer_sz - context->used_sz; + + /* Our buffer write position */ + uint8_t* wptr = context->buffer; + + /* + * Dumb approach for now, replace the entire buffer. Round + * the start and end points to block boundaries for efficiency. + */ + pos = filereader_get_block_pos(context, pos); + end = filereader_get_block_pos(context, end + (context->block_sz - 1)); + + /* Clamp the end position to the file size */ + if (end > context->file_sz) { + end = context->file_sz; + } + + /* Compute new rounded size, then clamp to buffer size */ + sz = end - pos; + if (sz > context->buffer_sz) { + sz = context->buffer_sz; + } + + /* Can we re-use existing data? */ + if ( + (pos >= context->pos) + && (pos < buffer_end) + && (end > buffer_end) + ) { + return filereader_append_buffer(context, end - buffer_end); + } else if ( + (pos < context->pos) + && (end >= context->pos) + && (end <= buffer_end) + ) { + return filereader_prepend_buffer(context, context->pos - pos); + } else { + /* Nope, read the lot in */ + const uint16_t file_rem = context->file_sz - pos; + if (file_rem < context->buffer_sz) { + sz = file_rem; + } else { + sz = context->buffer_sz; + } + + int read_res = filereader_read( + context, pos, sz, + context->buffer + ); + if (read_res >= 0) { + context->pos = pos; + context->used_sz = sz; + } + return read_res; + } + } else { + /* Nothing to do, we have the required data already */ + return 0; + } +} + +/** + * Try to read the data into the buffer, then return a pointer to it. + * + * \retval NULL The region could not be loaded into the buffer. + */ +static uint8_t *filereader_fetch_ptr( + struct filereader *context, uint16_t pos, uint16_t sz +) { + /* Ensure the data we need is present */ + if (filereader_load_buffer(context, pos, sz) < 0) { + /* We failed */ + return NULL; + } else { + return filereader_get_ptr(context, pos, sz); + } +} + +/** + * Fetch the reader context from the CborValue + */ +static struct filereader *filereader_get_context(const CborValue * const value) { + return (struct filereader*)(value->parser->data.ctx); +} + +/** + * Fetch the CborValue read position + */ +static uint16_t filereader_get_pos(const CborValue * const value) { + return (uint16_t)(uintptr_t)(value->source.token); +} + +/** + * Set the CborValue read position + */ +static void filereader_set_pos(CborValue * const value, uint16_t new_pos) { + value->source.token = (void*)(uintptr_t)new_pos; +} + +/** + * Return `true` if there is at least \a len bytes that can be read from + * the file at this moment in time. + */ +static bool filereader_impl_can_read_bytes( + const struct CborValue *value, + size_t len +) { + const struct filereader *context = filereader_get_context(value); + const uint16_t pos = filereader_get_pos(value); + + return ((size_t)pos + len) <= context->file_sz; +} + +/** + * Read the bytes from the buffer without advancing the read pointer. + */ +static void* filereader_impl_read_bytes( + const struct CborValue *value, + void* dst, size_t offset, size_t len +) { + struct filereader *context = filereader_get_context(value); + + /* Determine read position factoring in offset */ + const uint16_t pos = filereader_get_pos(value) + offset; + + /* Fetch the data from the file */ + const uint8_t* ptr = filereader_fetch_ptr(context, pos, (uint16_t)len); + if (ptr != NULL) { + return memcpy(dst, ptr, len); + } else { + /* We could not read the data */ + return NULL; + } +} + +/** + * Advance the pointer by the requested amount. + */ +static void filereader_impl_advance_bytes(struct CborValue *value, size_t len) { + filereader_set_pos(value, filereader_get_pos(value) + (uint16_t)len); +} + +/** + * Retrieve a pointer to the string defined by the given offset and length. + */ +CborError filereader_impl_transfer_string( + struct CborValue *value, + const void **userptr, size_t offset, size_t len +) { + struct filereader *context = filereader_get_context(value); + + /* Determine read position factoring in offset */ + const uint16_t pos = filereader_get_pos(value) + offset; + + /* Fetch the data from the file */ + const uint8_t* ptr = filereader_fetch_ptr(context, pos, (uint16_t)len); + if (ptr != NULL) { + /* All good, advance the cursor past the data and return the pointer */ + filereader_set_pos(value, pos + len); + *userptr = (void*)ptr; + return CborNoError; + } else { + /* We could not read the data */ + return CborErrorIO; + } +} + +/** + * Implementation of the CBOR File Reader operations. + */ +static const struct CborParserOperations filereader_ops = { + .can_read_bytes = filereader_impl_can_read_bytes, + .read_bytes = filereader_impl_read_bytes, + .advance_bytes = filereader_impl_advance_bytes, + .transfer_string = filereader_impl_transfer_string +}; + +/** + * Open a CBOR file for reading. + * + * \param[inout] parser CBOR parser object to initialise. + * \param[inout] value Root CBOR cursor object to initialise. + * + * \param[inout] context The file reader context. This must exist + * for the duration the file is open. + * + * \param[inout] buffer Read buffer allocated by the caller where + * the read data will be stored. + * + * \param[in] buffer_sz Size of the read buffer. + * + * \param[in] path The path to the file being read. + * + * \param[in] flags `open` flags. `O_RDONLY` is logic-ORed + * with this value, but the user may provide + * other options here. + * + * \param[in] block_sz Size of read blocks. Where possible, + * reads will be rounded up and aligned with + * blocks of this size for efficiency. Set + * to 0 to default to `buffer_sz / 2`. + * + * \retval CborErrorIO The `open` call failed for some reason, + * see the POSIX standard `errno` variable + * for why. + * + * \retval CborErrorDataTooLarge The CBOR document is too big to be + * handled by this reader. + * + * \retval CborNoError CBOR encoder initialised successfully. + */ +CborError filereader_open( + CborParser * const parser, + CborValue * const value, + struct filereader * const context, + uint8_t *buffer, + uint16_t buffer_sz, + const char* path, + int flags, + uint16_t block_sz +) +{ + CborError error = CborNoError; + struct stat path_stat; + + /* Determine the file size */ + if (stat(path, &path_stat) < 0) { + /* stat fails */ + error = CborErrorIO; + } else { + context->fd = open(path, O_RDONLY | flags); + if (context->fd < 0) { + /* Open fails */ + error = CborErrorIO; + } else { + /* Sanity check document size */ + if (path_stat.st_size > UINT16_MAX) { + error = CborErrorDataTooLarge; + } else { + /* Initialise structure */ + context->pos = 0; + context->used_sz = 0; + context->buffer = buffer; + context->buffer_sz = buffer_sz; + context->file_sz = (uint16_t)path_stat.st_size; + + if (block_sz == 0) { + block_sz = buffer_sz / 2; + } + context->block_sz = block_sz; + + /* Fill the initial buffer */ + if (filereader_load_buffer(context, 0, buffer_sz) >= 0) { + /* Initialise the CBOR parser */ + error = cbor_parser_init_reader( + &filereader_ops, parser, value, (void*)context + ); + } + } + + if (error != CborNoError) { + /* Close the file, if we can */ + assert(close(context->fd) == 0); + context->fd = -1; + } + } + } + + return error; +} + +/** + * Close the file reader. + * + * \param[inout] context File reader context to close. + * + * \retval CborErrorIO The `close` call failed for some + * reason, see the POSIX standard + * `errno` variable for why. + * + * \retval CborNoError File closed, `fd` should be set to -1. + */ +CborError filereader_close(struct filereader * const context) +{ + CborError error = CborNoError; + + /* Try to close the file */ + if (close(context->fd) < 0) { + /* Close fails! */ + error = CborErrorIO; + } else { + context->fd = -1; + } + + return error; +} + +/* --- Example usage of the above reader --- */ + +/** + * Indent the output text to the level specified. Taken from `simplereader.c` + */ +static void indent(int nestingLevel) +{ + while (nestingLevel--) + printf(" "); +} + +/** + * Dump the raw bytes given. Taken from `simplereader.c` + */ +static void dumpbytes(const uint8_t *buf, size_t len) +{ + while (len--) + printf("%02X ", *buf++); +} + +/** + * Recursively dump the CBOR data structure. Taken from `simplereader.c` + */ +static CborError dumprecursive(CborValue *it, int nestingLevel) +{ + while (!cbor_value_at_end(it)) { + CborError err; + CborType type = cbor_value_get_type(it); + + indent(nestingLevel); + switch (type) { + case CborArrayType: + case CborMapType: { + // recursive type + CborValue recursed; + assert(cbor_value_is_container(it)); + puts(type == CborArrayType ? "Array[" : "Map["); + err = cbor_value_enter_container(it, &recursed); + if (err) + return err; // parse error + err = dumprecursive(&recursed, nestingLevel + 1); + if (err) + return err; // parse error + err = cbor_value_leave_container(it, &recursed); + if (err) + return err; // parse error + indent(nestingLevel); + puts("]"); + continue; + } + + case CborIntegerType: { + int64_t val; + cbor_value_get_int64(it, &val); // can't fail + printf("%lld\n", (long long)val); + break; + } + + case CborByteStringType: { + uint8_t *buf; + size_t n; + err = cbor_value_dup_byte_string(it, &buf, &n, it); + if (err) + return err; // parse error + dumpbytes(buf, n); + printf("\n"); + free(buf); + continue; + } + + case CborTextStringType: { + char *buf; + size_t n; + err = cbor_value_dup_text_string(it, &buf, &n, it); + if (err) + return err; // parse error + puts(buf); + free(buf); + continue; + } + + case CborTagType: { + CborTag tag; + cbor_value_get_tag(it, &tag); // can't fail + printf("Tag(%lld)\n", (long long)tag); + break; + } + + case CborSimpleType: { + uint8_t type; + cbor_value_get_simple_type(it, &type); // can't fail + printf("simple(%u)\n", type); + break; + } + + case CborNullType: + puts("null"); + break; + + case CborUndefinedType: + puts("undefined"); + break; + + case CborBooleanType: { + bool val; + cbor_value_get_boolean(it, &val); // can't fail + puts(val ? "true" : "false"); + break; + } + + case CborDoubleType: { + double val; + if (false) { + float f; + case CborFloatType: + cbor_value_get_float(it, &f); + val = f; + } else { + cbor_value_get_double(it, &val); + } + printf("%g\n", val); + break; + } + case CborHalfFloatType: { + uint16_t val; + cbor_value_get_half_float(it, &val); + printf("__f16(%04x)\n", val); + break; + } + + case CborInvalidType: + assert(false); // can't happen + break; + } + + err = cbor_value_advance_fixed(it); + if (err) + return err; + } + return CborNoError; +} + +/** + * Print the error encountered. If the error is `CborErrorIO`, also check + * the global `errno` variable and print the resultant error seen. + * + * \param[in] error CBORError constant + */ +void print_err(CborError error) +{ + if (error == CborErrorIO) { + printf("IO: %s\n", strerror(errno)); + } else { + printf("%s\n", cbor_error_string(error)); + } +} + +int main(int argc, char **argv) +{ + if (argc < 2) { + printf( + "Usage: %s [buffer_sz [block_sz]]\n", + argv[0] + ); + return 1; + } else { + struct filereader context; + CborParser parser; + CborValue value; + CborError error; + + uint16_t buffer_sz = 64; + uint16_t block_sz = 0; + + if (argc > 2) { + /* buffer_sz given */ + char *endptr = NULL; + unsigned long long_buffer_sz = strtoul(argv[2], &endptr, 0); + + if (!endptr || *endptr) { + printf("Invalid buffer size %s\n", argv[2]); + return 1; + } + + if (long_buffer_sz > UINT16_MAX) { + printf("Buffer size (%lu bytes) too big\n", long_buffer_sz); + return 1; + } + + buffer_sz = (uint16_t)long_buffer_sz; + + if (argc > 3) { + /* block_sz given */ + char *endptr = NULL; + unsigned long long_block_sz = strtoul(argv[3], &endptr, 0); + + if (!endptr || *endptr) { + printf("Invalid block size %s\n", argv[3]); + return 1; + } + + if (long_block_sz > buffer_sz) { + printf("Block size (%lu bytes) too big\n", long_block_sz); + return 1; + } + + block_sz = (uint16_t)long_block_sz; + } + } + + /* Allocate the buffer on the stack */ + uint8_t buffer[buffer_sz]; + + /* Open the file for writing, create if needed */ + error = filereader_open( + &parser, /* CBOR context */ + &value, /* CBOR cursor */ + &context, /* Reader context */ + buffer, buffer_sz, /* Reader buffer & size */ + argv[1], /* File name */ + 0, /* Open flags */ + block_sz /* Block size */ + ); + + if (error != CborNoError) { + printf("Failed to open %s for reading: ", argv[1]); + print_err(error); + } else { + error = dumprecursive(&value, 0); + if (error != CborNoError) { + printf("Failed to read file: "); + print_err(error); + } + + error = filereader_close(&context); + if (error != CborNoError) { + printf("Failed to close file: "); + print_err(error); + } + } + + if (error != CborNoError) { + return 2; + } else { + return 0; + } + } +} diff --git a/examples/bufferedwriter.c b/examples/bufferedwriter.c new file mode 100644 index 00000000..747fbbb9 --- /dev/null +++ b/examples/bufferedwriter.c @@ -0,0 +1,711 @@ +/* vim: set sw=4 ts=4 et tw=78: */ + +/** + * \brief An example of a buffered CBOR file writer using low-level + * POSIX file I/O as might be implemented in a microcontroller + * RTOS. + * + * \author Stuart Longland + * + * \copyright tinycbor project contributors + * + * \file bufferedwriter.c + */ + +/* Includes for POSIX low-level file I/O */ +#include +#include +#include + +/* Pull in "standard" integer types */ +#include + +/* Pull in definitions for printf and errno */ +#include +#include +#include + +/* For example usage */ +#include + +#include "../src/cbor.h" + +/** + * File writer buffer size. This should be tuned to balance memory usage and + * performance. Most interfaces, bigger writes are more efficient, but on a + * small MCU, memory may be tight. + * + * We're using `uint8_t` to represent our buffer position, so this must be + * strictly less than 256 bytes unless you change it in \ref filewriter (and + * in \ref filewriter_writer_impl) below. + */ +#define FILEWRITER_BUFFER_SZ (64) + +/** + * Context for the file writer. This stores the file descriptor, the write + * buffer, and a counter indicating our position within it. + */ +struct filewriter +{ + /** + * Write buffer. `tinycbor` writes will be initially buffered here, and + * the buffer will automatically be flushed: + * - when the buffer position counter reaches \ref FILEWRITER_BUFFER_SZ + * - when the file is closed with \ref filewriter_close + */ + uint8_t buffer[FILEWRITER_BUFFER_SZ]; + + /** + * File descriptor, returned by the `open` system call. + */ + int fd; + + /** + * Position within the buffer. When less than \ref FILEWRITER_BUFFER_SZ + * this indicates the position for new data. If new data arrives and + * this value is equal to \ref FILEWRITER_BUFFER_SZ, the buffer will be + * flushed first before writing. + */ + uint8_t pos; +}; + +/* Forward declaration, we'll cover this later */ +static CborError filewriter_writer_impl( + void* token, const void* data, size_t len, CborEncoderAppendType append +); + +/** + * Open a CBOR file for writing. + * + * \param[inout] encoder CBOR encoder object to initialise. + * + * \param[inout] context The file writer context. This must exist + * for the duration the file is open. + * + * \param[in] path The path to the file being written. + * + * \param[in] flags `open` flags. `O_WRONLY` is logic-ORed + * with this value, but the user may provide + * other options here. + * + * \param[in] mode Mode bits to set on the created file. + * + * \retval CborErrorIO The `open` call failed for some reason, + * see the POSIX standard `errno` variable + * for why. + * + * \retval CborNoError CBOR encoder initialised successfully. + */ +CborError filewriter_open( + CborEncoder * const encoder, + struct filewriter * const context, + const char* path, + int flags, + mode_t mode +) +{ + CborError error = CborNoError; + + context->fd = open(path, O_WRONLY | flags, mode); + if (context->fd < 0) { + /* Open fails */ + error = CborErrorIO; + } else { + /* Initialise structure */ + context->pos = 0; + + /* Initialise the CBOR encoder */ + cbor_encoder_init_writer(encoder, filewriter_writer_impl, context); + } + + return error; +} + +/** + * Explicitly flush the content of the buffer. This is called automatically + * when the file is closed or when we run out of buffer space. Is a no-op if + * there is nothing to flush. + * + * \param[inout] context File writer context to flush. + * + * \retval CborErrorIO The `write` call failed for some reason, + * see the POSIX standard `errno` variable + * for why. + * + * \retval CborNoError Buffer flushed, position reset. + */ +CborError filewriter_flush(struct filewriter * const context) +{ + CborError error = CborNoError; + + if (context->pos > 0) { + if (write(context->fd, context->buffer, context->pos) < context->pos) { + error = CborErrorIO; + } else { + /* Success */ + context->pos = 0; + } + } + + return error; +} + +/** + * Close the file writer, flushing any remaining data and finishing the write. + * It is assumed that relevant CBOR containers have been closed first. + * + * \param[inout] context File writer context to flush. + * + * \retval CborErrorIO The `write` or `close` call failed for + * some reason, see the POSIX standard + * `errno` variable for why. (CBOR file + * should be considered invalid in this + * case.) *The file may still be open!* + * + * \retval CborNoError File closed, `fd` should be set to -1. + */ +CborError filewriter_close(struct filewriter * const context) +{ + CborError error = filewriter_flush(context); + + if (error == CborNoError) { + int res = close(context->fd); + if (res < 0) { + /* Close failed */ + error = CborErrorIO; + } else { + /* Mark context as closed */ + context->fd = -1; + } + } + + return error; +} + +/** + * CBOR Writer implementation. This function implements the necessary + * interface expected by `tinycbor` to perform synchronous writes to a + * file arbitrarily. Flushing is automatically handled. + */ +static CborError filewriter_writer_impl( + void* token, const void* data, size_t len, CborEncoderAppendType append +) +{ + struct filewriter* context = (struct filewriter*)token; + const uint8_t* rptr = (const uint8_t*)data; + CborError error = CborNoError; + + (void)append; /* We don't use the `append` argument */ + + while ((len > 0) && (error == CborNoError)) { + /* How much space is left? */ + uint8_t rem = FILEWRITER_BUFFER_SZ - context->pos; + + /* Is there any space? */ + if (rem > 0) { + /* Where is our write pointer at? */ + uint8_t* wptr = &(context->buffer[context->pos]); + + /* How much can we write? */ + uint8_t sz = rem; + if (sz > len) { + /* Clamp to amount of data available */ + sz = len; + } + + /* Copy that into the buffer */ + memcpy(wptr, rptr, sz); + context->pos += sz; + rptr += sz; + len -= sz; + rem -= sz; + } + + /* Are we full yet? */ + if (rem == 0) { + error = filewriter_flush(context); + } + } + + return error; +} + +/* --- Example usage of the above writer --- */ + +/** + * Print the error encountered. If the error is `CborErrorIO`, also check + * the global `errno` variable and print the resultant error seen. + * + * \param[in] error CBORError constant + */ +void print_err(CborError error) +{ + if (error == CborErrorIO) { + printf("IO: %s\n", strerror(errno)); + } else { + printf("%s\n", cbor_error_string(error)); + } +} + +/* Forward declarations */ +int exec_arg_array(CborEncoder * const encoder, size_t len, int argc, char **argv); +int exec_arg_map(CborEncoder * const encoder, size_t len, int argc, char **argv); + +/** + * Interpret the arguments given and execute one of the `tinycbor` routines. + * + * \param[inout] encoder CBOREncoder instance + * \param[in] argc Number of command line arguments remaining + * \param[in] argv Command line arguments' values + * + * \retval ≤0 CBOR error occurred, stop here. + * \retval >0 Number of arguments consumed. + */ +int exec_arg(CborEncoder * const encoder, int argc, char **argv) +{ + if (argc > 1) { + CborError error; + int consumed; + int len = strlen(argv[0]); + + printf("Command: %s (%d bytes)\n", argv[0], len); + if (len == 1) { + /* Single-character commands */ + switch (argv[0][0]) { + case '{': /* Begin unknown-length map */ + consumed = exec_arg_map( + encoder, CborIndefiniteLength, + argc - 1, argv + 1 + ); + if (consumed > 0) { + consumed++; + } + return consumed; + case '[': /* Begin unknown-length array */ + consumed = exec_arg_array( + encoder, CborIndefiniteLength, + argc - 1, argv + 1 + ); + if (consumed > 0) { + consumed++; + } + return consumed; + + case 'N': /* Null */ + case 'n': + error = cbor_encode_null(encoder); + if (error != CborNoError) { + printf("Failed at null: "); + print_err(error); + return -1; + } + return 1; + case 'U': /* Undefined */ + case 'u': + error = cbor_encode_undefined(encoder); + if (error != CborNoError) { + printf("Failed at undefined: "); + print_err(error); + return -1; + } + return 1; + case 'F': /* False */ + case 'f': + error = cbor_encode_boolean(encoder, false); + if (error != CborNoError) { + printf("Failed at false: "); + print_err(error); + return -1; + } + return 1; + case 'T': /* True */ + case 't': + error = cbor_encode_boolean(encoder, true); + if (error != CborNoError) { + printf("Failed at true: "); + print_err(error); + return -1; + } + return 1; + default: + printf("Unknown single-character command: %s", argv[0]); + return -1; + } + } else if (strncmp(argv[0], "map(", 4) == 0) { + /* Fixed-size map */ + char *endptr = NULL; + unsigned long maplen = strtoul(&(argv[0][4]), &endptr, 0); + + if (!endptr || (*endptr != ')')) { + /* Not a valid length */ + printf("Invalid length for map: %s\n", argv[0]); + return -1; + } else { + consumed = exec_arg_map(encoder, maplen, argc - 1, argv + 1); + if (consumed > 0) { + consumed++; + } + return consumed; + } + } else if (strncmp(argv[0], "array(", 6) == 0) { + /* Fixed-size array */ + char *endptr = NULL; + unsigned long arraylen = strtoul(&(argv[0][4]), &endptr, 0); + + if (!endptr || (*endptr != ')')) { + /* Not a valid length */ + printf("Invalid length for array: %s\n", argv[0]); + return -1; + } else { + consumed = exec_arg_array( + encoder, arraylen, argc - 1, argv + 1 + ); + if (consumed > 0) { + consumed++; + } + return consumed; + } + } else if (argv[0][0] == 's') { + /* Text string */ + error = cbor_encode_text_string(encoder, &(argv[0][1]), len - 1); + if (error != CborNoError) { + printf( + "Failed at text string (%s): ", + argv[0] + ); + print_err(error); + return -1; + } else { + return 1; + } + } else if (argv[0][0] == 'x') { + /* Byte string, total length must be odd with 'x' prefix */ + if ((len % 2) == 0) { + printf("Byte string must be an even number of hex digits.\n"); + return -1; + } + + uint8_t bytes[len / 2]; + int i; + for (i = 1; i < len; i++) { + char nybble = argv[0][i]; + if ((nybble >= '0') && (nybble <= '9')) { + nybble -= '0'; + } else if ((nybble >= 'A') && (nybble <= 'F')) { + nybble -= 'A'; + nybble += 10; + } else if ((nybble >= 'a') && (nybble <= 'f')) { + nybble -= 'a'; + nybble += 10; + } else { + printf("Unsupported character '%c' in byte string at %d\n", + nybble, i); + return -1; + } + + if ((i % 2) == 0) { + /* Even numbered: lower nybble */ + bytes[(i - 1)/2] |= nybble; + } else { + /* Odd numbered: upper nybble */ + bytes[i/2] = nybble << 4; + } + } + + error = cbor_encode_byte_string(encoder, bytes, len / 2); + if (error != CborNoError) { + printf( + "Failed at byte string (%s): ", + argv[0] + ); + print_err(error); + return -1; + } else { + return 1; + } + } else if (argv[0][0] == 'd') { + /* 32-bit float number */ + char *endptr = NULL; + double d = strtod(&(argv[0][1]), &endptr); + + if (!endptr || (*endptr)) { + /* Invalid number */ + printf("Invalid double %s\n", argv[0]); + return -1; + } + + error = cbor_encode_double(encoder, d); + if (error != CborNoError) { + printf( + "Failed at double (%s): ", + argv[0] + ); + print_err(error); + return -1; + } else { + return 1; + } + } else if (argv[0][0] == 'f') { + /* 32-bit float number */ + char *endptr = NULL; + float f = strtof(&(argv[0][1]), &endptr); + + if (!endptr || (*endptr)) { + /* Invalid number */ + printf("Invalid float %s\n", argv[0]); + return -1; + } + + error = cbor_encode_float(encoder, f); + if (error != CborNoError) { + printf( + "Failed at float (%s): ", + argv[0] + ); + print_err(error); + return -1; + } else { + return 1; + } + } else if (argv[0][0] == 'u') { + /* Unsigned integer (positive) number */ + char *endptr = NULL; + unsigned long long ull = strtoull(&(argv[0][1]), &endptr, 0); + + if (!endptr || (*endptr)) { + /* Invalid number */ + printf("Invalid unsigned integer %s\n", argv[0]); + return -1; + } + + error = cbor_encode_uint(encoder, ull); + if (error != CborNoError) { + printf( + "Failed at unsigned integer (%s): ", + argv[0] + ); + print_err(error); + return -1; + } else { + return 1; + } + } else if (argv[0][0] == '-') { + /* Unsigned integer (negative) number */ + char *endptr = NULL; + unsigned long long ull = strtoull(&(argv[0][1]), &endptr, 0); + + if (!endptr || (*endptr)) { + /* Invalid number */ + printf("Invalid negative unsigned integer %s\n", argv[0]); + return -1; + } + + error = cbor_encode_negative_int(encoder, ull); + if (error != CborNoError) { + printf( + "Failed at negative unsigned integer (%s): ", + argv[0] + ); + print_err(error); + return -1; + } else { + return 1; + } + } else { + printf("Unknown command: %s", argv[0]); + return -1; + } + } else { + /* No arguments to consume. */ + printf("End of arguments.\n"); + return 0; + } +} + +/** + * Interpret the arguments given and execute one of the `tinycbor` routines, + * in an array context. + * + * \param[inout] encoder CBOREncoder instance + * \param[in] len Length of the array. + * \param[in] argc Number of command line arguments remaining + * \param[in] argv Command line arguments' values + * + * \retval ≤0 CBOR error occurred, stop here. + * \retval >0 Number of arguments consumed. + */ +int exec_arg_array(CborEncoder * const encoder, size_t len, int argc, char **argv) { + int consumed = 0; + CborEncoder container; + CborError error; + + error = cbor_encoder_create_array(encoder, &container, len); + if (error != CborNoError) { + printf("Failed to create array (length=%lu): ", len); + print_err(error); + consumed = -1; + } else { + while ((consumed >= 0) && (argc > 0) && (argv[0][0] != ']')) { + int arg_consumed = exec_arg(&container, argc, argv); + + if (arg_consumed > 0) { + consumed += arg_consumed; + argc -= arg_consumed; + argv += arg_consumed; + } else { + /* Error condition */ + printf( + "Failed inside array context (after %d arguments).\n", + consumed + ); + consumed = -1; + } + } + + if (consumed >= 0) { + printf("Close array after %d arguments\n", consumed); + + /* Count end-of-array */ + consumed++; + + error = cbor_encoder_close_container(encoder, &container); + if (error != CborNoError) { + printf("Failed to finish array (length=%lu): ", len); + print_err(error); + consumed = -1; + } + } + } + + return consumed; +} + +/** + * Interpret the arguments given and execute one of the `tinycbor` routines, + * in a map context. + * + * \param[inout] encoder CBOREncoder instance + * \param[in] len Length of the map. + * \param[in] argc Number of command line arguments remaining + * \param[in] argv Command line arguments' values + * + * \retval ≤0 CBOR error occurred, stop here. + * \retval >0 Number of arguments consumed. + */ +int exec_arg_map(CborEncoder * const encoder, size_t len, int argc, char **argv) { + int consumed = 0; + CborEncoder container; + CborError error; + + error = cbor_encoder_create_map(encoder, &container, len); + if (error != CborNoError) { + printf("Failed to create map (length=%lu): ", len); + print_err(error); + consumed = -1; + } else { + while ((consumed >= 0) && (argc > 0) && (argv[0][0] != '}')) { + int arg_consumed = exec_arg(&container, argc, argv); + + if (arg_consumed > 0) { + consumed += arg_consumed; + argc -= arg_consumed; + argv += arg_consumed; + } else { + /* Error condition */ + printf( + "Failed inside map context (after %d arguments).\n", + consumed + ); + consumed = -1; + } + } + + if (consumed >= 0) { + printf("Close map after %d arguments\n", consumed); + + /* Count end-of-map */ + consumed++; + + error = cbor_encoder_close_container(encoder, &container); + if (error != CborNoError) { + printf("Failed to finish map (length=%lu): ", len); + print_err(error); + consumed = -1; + } + } + } + + return consumed; +} + +int main(int argc, char **argv) +{ + if (argc < 2) { + printf( + "Usage: %s ...\n" + "Valid commands:\n" + "\t{\tStart an unknown-length map\n" + "\t[\tStart an unknown-length array\n" + "\tmap() {\tStart a map of length \n" + "\tarray() [\tStart an array of length \n" + "\ts\tInsert a text string\n" + "\tx\tInsert a byte string\n" + "\tu\tInsert an unsigned positive integer\n" + "\t-\tInsert an unsigned negative integer\n" + "\td\tInsert a 64-bit float\n" + "\tf\tInsert a 32-bit float\n" + "\tf, t\tInsert FALSE or TRUE (case insensitive)\n" + "\tn, u\tInsert NULL or UNDEFINED (case insensitive)\n" + "\nInside maps:\n" + "\t}\tEnd the current map\n" + "\nInside arrays:\n" + "\t]\tEnd the current array\n", + argv[0] + ); + return 1; + } else { + struct filewriter context; + CborEncoder encoder; + CborError error; + + /* Open the file for writing, create if needed */ + error = filewriter_open( + &encoder, /* CBOR context */ + &context, /* Writer context */ + argv[1], /* File name */ + O_CREAT, /* Open flags */ + S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH /* File permissions */ + ); + + if (error != CborNoError) { + printf("Failed to open %s for writing: ", argv[1]); + print_err(error); + } else { + argv += 2; + argc -= 2; + + while (argc > 0) { + int consumed = exec_arg(&encoder, argc, argv); + + if (consumed > 0) { + argc -= consumed; + argv += consumed; + } else { + break; + } + } + + error = filewriter_close(&context); + if (error != CborNoError) { + printf("Failed to close file: "); + print_err(error); + } + } + + if (error != CborNoError) { + return 2; + } else { + return 0; + } + } +} diff --git a/src/cbor.h b/src/cbor.h index 86909d58..430ef217 100644 --- a/src/cbor.h +++ b/src/cbor.h @@ -216,7 +216,7 @@ typedef enum CborEncoderAppendType CborEncoderApendRawData = 2 } CborEncoderAppendType; -typedef CborError (*CborEncoderWriteFunction)(void *, const void *, size_t, CborEncoderAppendType); +typedef CborError (*CborEncoderWriteFunction)(void *token, const void *data, size_t len, CborEncoderAppendType append); enum CborEncoderFlags { @@ -321,20 +321,23 @@ enum CborParserIteratorFlags }; struct CborValue; + + struct CborParserOperations { - bool (*can_read_bytes)(void *token, size_t len); - void *(*read_bytes)(void *token, void *dst, size_t offset, size_t len); - void (*advance_bytes)(void *token, size_t len); - CborError (*transfer_string)(void *token, const void **userptr, size_t offset, size_t len); + bool (*can_read_bytes)(const struct CborValue *value, size_t len); + void *(*read_bytes)(const struct CborValue *value, void *dst, size_t offset, size_t len); + void (*advance_bytes)(struct CborValue *value, size_t len); + CborError (*transfer_string)(struct CborValue *value, const void **userptr, size_t offset, size_t len); }; struct CborParser { union { const uint8_t *end; - const struct CborParserOperations *ops; - } source; + void *ctx; + } data; + const struct CborParserOperations *ops; enum CborParserGlobalFlags flags; }; typedef struct CborParser CborParser; diff --git a/src/cborencoder.c b/src/cborencoder.c index 69aebf40..b5d7a7ca 100644 --- a/src/cborencoder.c +++ b/src/cborencoder.c @@ -196,6 +196,23 @@ * Structure used to encode to CBOR. */ +/** + * \file cbor.h + * \typedef CborEncoderWriteFunction + * + * Writer interface call-back function. When there is data to be written to + * the CBOR document, this routine will be called. The \a token parameter is + * taken from the \a token argument provided to \ref cbor_encoder_init_writer + * and may be used in any way the writer function sees fit. + * + * The \a data parameter contains a pointer to the raw bytes to be copied to + * the output buffer, with \a len specifying how long the payload is, which + * can be as small as a single byte or an entire (byte or text) string. + * + * The \a append parameter informs the writer function whether it is writing + * a string or general CBOR data. + */ + /** * Initializes a CborEncoder structure \a encoder by pointing it to buffer \a * buffer of size \a size. The \a flags field is currently unused and must be diff --git a/src/cborinternal_p.h b/src/cborinternal_p.h index 19273acd..207f9ae8 100644 --- a/src/cborinternal_p.h +++ b/src/cborinternal_p.h @@ -203,9 +203,9 @@ static inline bool can_read_bytes(const CborValue *it, size_t n) if (CBOR_PARSER_READER_CONTROL >= 0) { if (it->parser->flags & CborParserFlag_ExternalSource || CBOR_PARSER_READER_CONTROL != 0) { #ifdef CBOR_PARSER_CAN_READ_BYTES_FUNCTION - return CBOR_PARSER_CAN_READ_BYTES_FUNCTION(it->source.token, n); + return CBOR_PARSER_CAN_READ_BYTES_FUNCTION(it, n); #else - return it->parser->source.ops->can_read_bytes(it->source.token, n); + return it->parser->ops->can_read_bytes(it, n); #endif } } @@ -213,7 +213,7 @@ static inline bool can_read_bytes(const CborValue *it, size_t n) /* Convert the pointer subtraction to size_t since end >= ptr * (this prevents issues with (ptrdiff_t)n becoming negative). */ - return (size_t)(it->parser->source.end - it->source.ptr) >= n; + return (size_t)(it->parser->data.end - it->source.ptr) >= n; } static inline void advance_bytes(CborValue *it, size_t n) @@ -221,9 +221,9 @@ static inline void advance_bytes(CborValue *it, size_t n) if (CBOR_PARSER_READER_CONTROL >= 0) { if (it->parser->flags & CborParserFlag_ExternalSource || CBOR_PARSER_READER_CONTROL != 0) { #ifdef CBOR_PARSER_ADVANCE_BYTES_FUNCTION - CBOR_PARSER_ADVANCE_BYTES_FUNCTION(it->source.token, n); + CBOR_PARSER_ADVANCE_BYTES_FUNCTION(it, n); #else - it->parser->source.ops->advance_bytes(it->source.token, n); + it->parser->ops->advance_bytes(it, n); #endif return; } @@ -237,9 +237,9 @@ static inline CborError transfer_string(CborValue *it, const void **ptr, size_t if (CBOR_PARSER_READER_CONTROL >= 0) { if (it->parser->flags & CborParserFlag_ExternalSource || CBOR_PARSER_READER_CONTROL != 0) { #ifdef CBOR_PARSER_TRANSFER_STRING_FUNCTION - return CBOR_PARSER_TRANSFER_STRING_FUNCTION(it->source.token, ptr, offset, len); + return CBOR_PARSER_TRANSFER_STRING_FUNCTION(it, ptr, offset, len); #else - return it->parser->source.ops->transfer_string(it->source.token, ptr, offset, len); + return it->parser->ops->transfer_string(it, ptr, offset, len); #endif } } @@ -258,9 +258,9 @@ static inline void *read_bytes_unchecked(const CborValue *it, void *dst, size_t if (CBOR_PARSER_READER_CONTROL >= 0) { if (it->parser->flags & CborParserFlag_ExternalSource || CBOR_PARSER_READER_CONTROL != 0) { #ifdef CBOR_PARSER_READ_BYTES_FUNCTION - return CBOR_PARSER_READ_BYTES_FUNCTION(it->source.token, dst, offset, n); + return CBOR_PARSER_READ_BYTES_FUNCTION(it, dst, offset, n); #else - return it->parser->source.ops->read_bytes(it->source.token, dst, offset, n); + return it->parser->ops->read_bytes(it, dst, offset, n); #endif } } diff --git a/src/cborparser.c b/src/cborparser.c index 75a40888..739f1903 100644 --- a/src/cborparser.c +++ b/src/cborparser.c @@ -141,6 +141,75 @@ * \endif */ +/** + * \struct CborParserOperations + * + * Defines an interface for abstract document readers. This structure is used + * in conjunction with \ref cbor_parser_init_reader to define how the various + * required operations are to be implemented. + * + * + * \var CborParserOperations::can_read_bytes + * + * Determines whether \a len bytes may be read from the reader. This is + * called before \ref read_bytes and \ref transfer_bytes to ensure it is safe + * to read the requested number of bytes from the reader. + * + * \param value The CBOR value being parsed. + * + * \param len The number of bytes sought. + * + * \retval true \a len bytes may be read from the reader. + * \retval false Insufficient data is available to be read at this time. + * + * + * \var CborParserOperations::read_bytes + * + * Reads \a len bytes from the reader starting at \a offset bytes from + * the current read position and copies them to \a dst. The read pointer + * is *NOT* modified by this operation. + * + * \param value The CBOR value being parsed. + * + * \param dst The buffer the read bytes will be copied to. + * + * \param offset The starting position for the read relative to the + * current read position. + * + * \param len The number of bytes sought. + * + * + * \var CborParserOperations::advance_bytes + * + * Skips past \a len bytes from the reader without reading them. The read + * pointer is advanced in the process. + * + * \param value The CBOR value being parsed. + * + * \param len The number of bytes skipped. + * + * + * \var CborParserOperations::transfer_string + * + * Overwrite the user-supplied pointer \a userptr with the address where the + * data indicated by \a offset is located, then advance the read pointer + * \a len bytes beyond that point. + * + * This routine is used for accessing strings embedded in CBOR documents + * (both text and binary strings). + * + * \param value The CBOR value being parsed. + * + * \param userptr The pointer that will be updated to reference the location + * of the data in the buffer. + * + * \param offset The starting position for the read relative to the + * current read position. + * + * \param len The number of bytes sought. + */ + + static uint64_t extract_number_and_advance(CborValue *it) { /* This function is only called after we've verified that the number @@ -332,6 +401,14 @@ uint64_t _cbor_value_decode_int64_internal(const CborValue *value) return read_uint32(value, 1); } +static void cbor_parser_init_common(CborParser *parser, CborValue *it) +{ + memset(parser, 0, sizeof(*parser)); + it->parser = parser; + it->remaining = 1; /* there's one type altogether, usually an array or map */ + it->flags = 0; +} + /** * Initializes the CBOR parser for parsing \a size bytes beginning at \a * buffer. Parsing will use flags set in \a flags. The iterator to the first @@ -344,24 +421,39 @@ uint64_t _cbor_value_decode_int64_internal(const CborValue *value) */ CborError cbor_parser_init(const uint8_t *buffer, size_t size, uint32_t flags, CborParser *parser, CborValue *it) { - memset(parser, 0, sizeof(*parser)); - parser->source.end = buffer + size; + cbor_parser_init_common(parser, it); + parser->data.end = buffer + size; parser->flags = (enum CborParserGlobalFlags)flags; - it->parser = parser; it->source.ptr = buffer; - it->remaining = 1; /* there's one type altogether, usually an array or map */ - it->flags = 0; return preparse_value(it); } -CborError cbor_parser_init_reader(const struct CborParserOperations *ops, CborParser *parser, CborValue *it, void *token) +/** + * Initializes the CBOR parser for parsing a document that is read by an + * abstract reader interface defined by \a ops. The iterator to the first + * element is returned in \a it. + * + * The \a parser structure needs to remain valid throughout the decoding + * process. It is not thread-safe to share one CborParser among multiple + * threads iterating at the same time, but the object can be copied so multiple + * threads can iterate. + * + * The \a ops structure defines functions that implement the read process from + * the buffer given, see \ref CborParserOperations for further details. + * + * The \a ctx is stored in the \ref CborParser object as `data.ctx` and may be + * used however the reader implementation sees fit. For cursor-specific + * context information, the \ref CborValue `source.token` union member is + * initialised to `NULL` and may be used however the reader implementation + * sees fit. + */ +CborError cbor_parser_init_reader(const struct CborParserOperations *ops, CborParser *parser, CborValue *it, void *ctx) { - memset(parser, 0, sizeof(*parser)); - parser->source.ops = ops; + cbor_parser_init_common(parser, it); + parser->ops = ops; parser->flags = CborParserFlag_ExternalSource; - it->parser = parser; - it->source.token = token; - it->remaining = 1; + parser->data.ctx = ctx; + it->source.token = NULL; return preparse_value(it); } diff --git a/src/cborparser_dup_string.c b/src/cborparser_dup_string.c index 45cebc50..b6346177 100644 --- a/src/cborparser_dup_string.c +++ b/src/cborparser_dup_string.c @@ -35,14 +35,8 @@ #include "cbor.h" #include "compilersupport_p.h" +#include "memory.h" -#if defined(CBOR_CUSTOM_ALLOC_INCLUDE) -# include CBOR_CUSTOM_ALLOC_INCLUDE -#else -# include -# define cbor_malloc malloc -# define cbor_free free -#endif /** * \fn CborError cbor_value_dup_text_string(const CborValue *value, char **buffer, size_t *buflen, CborValue *next) diff --git a/src/cbortojson.c b/src/cbortojson.c index 5bcf3640..890769c7 100644 --- a/src/cbortojson.c +++ b/src/cbortojson.c @@ -35,6 +35,7 @@ #include "cborinternal_p.h" #include "compilersupport_p.h" #include "cborinternal_p.h" +#include #include #include @@ -179,7 +180,7 @@ static CborError dump_bytestring_base16(char **result, CborValue *it) return err; /* a Base16 (hex) output is twice as big as our buffer */ - buffer = (uint8_t *)malloc(n * 2 + 1); + buffer = (uint8_t *)cbor_malloc(n * 2 + 1); if (buffer == NULL) /* out of memory */ return CborErrorOutOfMemory; @@ -209,7 +210,7 @@ static CborError generic_dump_base64(char **result, CborValue *it, const char al /* a Base64 output (untruncated) has 4 bytes for every 3 in the input */ size_t len = (n + 5) / 3 * 4; - buffer = (uint8_t *)malloc(len + 1); + buffer = (uint8_t *)cbor_malloc(len + 1); if (buffer == NULL) /* out of memory */ return CborErrorOutOfMemory; @@ -395,7 +396,7 @@ static CborError tagged_value_to_json(FILE *out, CborValue *it, int flags, Conve if (err) return err; err = fprintf(out, "\"%s%s\"", pre, str) < 0 ? CborErrorIO : CborNoError; - free(str); + cbor_free(str); status->flags = TypeWasNotNative | TypeWasTagged | CborByteStringType; return err; } @@ -467,7 +468,7 @@ static CborError map_to_json(FILE *out, CborValue *it, int flags, ConversionStat /* first, print the key */ if (fprintf(out, "\"%s\":", key) < 0) { - free(key); + cbor_free(key); return CborErrorIO; } @@ -489,7 +490,7 @@ static CborError map_to_json(FILE *out, CborValue *it, int flags, ConversionStat } } - free(key); + cbor_free(key); if (err) return err; } @@ -568,7 +569,7 @@ static CborError value_to_json(FILE *out, CborValue *it, int flags, CborType typ if (err) return err; err = (fprintf(out, "\"%s\"", str) < 0) ? CborErrorIO : CborNoError; - free(str); + cbor_free(str); return err; } diff --git a/src/memory.h b/src/memory.h new file mode 100644 index 00000000..0032b93b --- /dev/null +++ b/src/memory.h @@ -0,0 +1,31 @@ +/**************************************************************************** +** +** Copyright (C) 2016 Intel Corporation +** +** Permission is hereby granted, free of charge, to any person obtaining a copy +** of this software and associated documentation files (the "Software"), to deal +** in the Software without restriction, including without limitation the rights +** to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +** copies of the Software, and to permit persons to whom the Software is +** furnished to do so, subject to the following conditions: +** +** The above copyright notice and this permission notice shall be included in +** all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +** OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +** THE SOFTWARE. +** +****************************************************************************/ + +#if defined(CBOR_CUSTOM_ALLOC_INCLUDE) +# include CBOR_CUSTOM_ALLOC_INCLUDE +#else +# include +# define cbor_malloc malloc +# define cbor_free free +#endif diff --git a/tests/parser/tst_parser.cpp b/tests/parser/tst_parser.cpp index 07333d5e..322e80ae 100644 --- a/tests/parser/tst_parser.cpp +++ b/tests/parser/tst_parser.cpp @@ -759,32 +759,32 @@ void tst_Parser::mapsAndArrays() "{_ 1: [_ " + expected + "], \"Hello\": {_ " + expected + ": (_ )}}"); } -struct Input { - QByteArray data; - int consumed; -}; - static const CborParserOperations byteArrayOps = { - /* can_read_bytes = */ [](void *token, size_t len) { - auto input = static_cast(token); - return input->data.size() - input->consumed >= int(len); + /* can_read_bytes = */ [](const CborValue *value, size_t len) { + QByteArray *data = static_cast(value->parser->data.ctx); + uintptr_t consumed = uintptr_t(value->source.token); + return uintptr_t(data->size()) - consumed >= uintptr_t(len); }, - /* read_bytes = */ [](void *token, void *dst, size_t offset, size_t len) { - auto input = static_cast(token); - return memcpy(dst, input->data.constData() + input->consumed + offset, len); + /* read_bytes = */ [](const CborValue *value, void *dst, size_t offset, size_t len) { + QByteArray *data = static_cast(value->parser->data.ctx); + uintptr_t consumed = uintptr_t(value->source.token); + return memcpy(dst, data->constData() + consumed + offset, len); }, - /* advance_bytes = */ [](void *token, size_t len) { - auto input = static_cast(token); - input->consumed += int(len); + /* advance_bytes = */ [](CborValue *value, size_t len) { + uintptr_t consumed = uintptr_t(value->source.token); + consumed += uintptr_t(len); + value->source.token = reinterpret_cast(consumed); }, - /* transfer_string = */ [](void *token, const void **userptr, size_t offset, size_t len) { + /* transfer_string = */ [](CborValue *value, const void **userptr, size_t offset, size_t len) { // ### - auto input = static_cast(token); - if (input->data.size() - input->consumed < int(len + offset)) + QByteArray *data = static_cast(value->parser->data.ctx); + uintptr_t consumed = uintptr_t(value->source.token); + if (uintptr_t(data->size()) - consumed < uintptr_t(len + offset)) return CborErrorUnexpectedEOF; - input->consumed += int(offset); - *userptr = input->data.constData() + input->consumed; - input->consumed += int(len); + consumed += uintptr_t(offset); + *userptr = data->constData() + consumed; + consumed += uintptr_t(len); + value->source.token = reinterpret_cast(consumed); return CborNoError; } }; @@ -794,11 +794,9 @@ void tst_Parser::readerApi() QFETCH(QByteArray, data); QFETCH(QString, expected); - Input input = { data, 0 }; - CborParser parser; CborValue first; - CborError err = cbor_parser_init_reader(&byteArrayOps, &parser, &first, &input); + CborError err = cbor_parser_init_reader(&byteArrayOps, &parser, &first, &data); QCOMPARE(err, CborNoError); QString decoded; @@ -807,7 +805,7 @@ void tst_Parser::readerApi() QCOMPARE(decoded, expected); // check we consumed everything - QCOMPARE(input.consumed, data.size()); + QCOMPARE(uintptr_t(first.source.token), uintptr_t(data.size())); } void tst_Parser::reparse_data() @@ -822,23 +820,23 @@ void tst_Parser::reparse() QFETCH(QByteArray, data); QFETCH(QString, expected); - Input input = { QByteArray(), 0 }; + QByteArray buffer; CborParser parser; CborValue first; - CborError err = cbor_parser_init_reader(&byteArrayOps, &parser, &first, &input); + CborError err = cbor_parser_init_reader(&byteArrayOps, &parser, &first, &buffer); QCOMPARE(err, CborErrorUnexpectedEOF); for (int i = 0; i < data.size(); ++i) { - input.data = data.left(i); + buffer = data.left(i); err = cbor_value_reparse(&first); if (err != CborErrorUnexpectedEOF) qDebug() << "At" << i; QCOMPARE(err, CborErrorUnexpectedEOF); - QCOMPARE(input.consumed, 0); + QCOMPARE(uintptr_t(first.source.token), 0U); } // now it should work - input.data = data; + buffer = data; err = cbor_value_reparse(&first); QCOMPARE(err, CborNoError); @@ -848,7 +846,7 @@ void tst_Parser::reparse() QCOMPARE(decoded, expected); // check we consumed everything - QCOMPARE(input.consumed, data.size()); + QCOMPARE(uintptr_t(first.source.token), uintptr_t(data.size())); } void tst_Parser::chunkedString_data() diff --git a/tests/tests.pro b/tests/tests.pro index 6036f0f9..627ffbc1 100644 --- a/tests/tests.pro +++ b/tests/tests.pro @@ -1,3 +1,3 @@ TEMPLATE = subdirs -SUBDIRS = parser encoder c90 cpp tojson +SUBDIRS = parser encoder cpp tojson msvc: SUBDIRS -= tojson