From: David Bremner <bremner@debian.org> This initial target use is in quoting queries for Xapian. We want to split into tokens, but preserve the delimiters between the tokens verbatim. --- util/string-util.c | 12 ++++++++++++ util/string-util.h | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/util/string-util.c b/util/string-util.c index b9039f4..1586483 100644 --- a/util/string-util.c +++ b/util/string-util.c @@ -34,6 +34,18 @@ strtok_len (char *s, const char *delim, size_t *len) return *len ? s : NULL; } +char * +strtok_len2 (char *s, const char *delim, size_t *len, size_t *delim_len) +{ + /* length of token */ + *len = strcspn (s, delim); + + /* length of following delimiter */ + *delim_len = strspn (s + *len, delim); + + return *len || *delim_len ? s : NULL; +} + int double_quote_str (void *ctx, const char *str, diff --git a/util/string-util.h b/util/string-util.h index 4fc7942..12398a5 100644 --- a/util/string-util.h +++ b/util/string-util.h @@ -19,6 +19,25 @@ char *strtok_len (char *s, const char *delim, size_t *len); +/* Like strtok_len, but return length of delimiters as well. Return + * value is indicated by pointer and length, not null terminator. + * Does _not_ skip initial delimiters. + * + * Usage pattern: + * + * const char *tok = input; + * const char *delim = " :.,"; + * size_t tok_len = 0; + * size_t delim_len = 0; + * + * while ((tok = strtok_len (tok + tok_len + delim_len, delim, + * &tok_len, &delim_len)) != NULL) { + * // do stuff with token and following delimiters. + * } + */ + +char *strtok_len2 (char *s, const char *delim, size_t *len, size_t *delim_len); + /* Copy str to dest, surrounding with double quotes. * Any internal double-quotes are doubled, i.e. a"b -> "a""b" * -- 1.7.10.4