From 32beb2bf5d6f51d46f3ed52995f04a72b1284c0f Mon Sep 17 00:00:00 2001 From: fooinha Date: 2017年12月21日 11:58:51 +0000 Subject: [PATCH 1/5] ngx-json-log: log to syslog --- README.md | 5 ++-- src/ngx_http_json_log_filter_module.c | 12 ++++++++ src/ngx_http_json_log_module.c | 33 ++++++++++++++++++++-- src/ngx_json_log_output.c | 40 +++++++++++++++++++++++++++ src/ngx_json_log_output.h | 19 +++++++++++-- src/ngx_stream_json_log_module.c | 37 +++++++++++++++++++++++-- 6 files changed, 136 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ab79b74..7c44140 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,13 @@ The output format is configurable. It also allows to log complex and multi-level JSON documents. -It supports logging to text file or to a kafka topic. +It supports logging to text file, to syslog or to a kafka topic. It supports multiple output destinations with multiple formats for a location. ### Current version and limitations -Current version released is 0.0.6. +Current version released is 0.0.7. Stream logging is only available when using nginx (>= 1.11.2). @@ -118,6 +118,7 @@ Additionally, the variable $http_json_err_log_req will be set with a base64 enco '; json_log file:/tmp/log my_log;' + json_log syslog:server=unix:/dev/log my_log;' ``` This will produce the following JSON line to '/tmp/log' file . diff --git a/src/ngx_http_json_log_filter_module.c b/src/ngx_http_json_log_filter_module.c index dbb0883..5aa3434 100644 --- a/src/ngx_http_json_log_filter_module.c +++ b/src/ngx_http_json_log_filter_module.c @@ -437,6 +437,18 @@ ngx_http_json_log_header_bad_request(ngx_http_request_t *r) continue; } + /* Write to syslog */ + if (location->type == NGX_JSON_LOG_SINK_SYSLOG) { + if (!location->syslog) { + continue; + } + if (ngx_json_log_write_sink_syslog(r->pool->log, + r->pool, location->syslog, txt) == NGX_ERROR) { + ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, "Syslog write error!"); + } + continue; + } + #if (NGX_HAVE_LIBRDKAFKA) /* Write to kafka */ if (location->type == NGX_JSON_LOG_SINK_KAFKA) { diff --git a/src/ngx_http_json_log_module.c b/src/ngx_http_json_log_module.c index d712a0b..2806aef 100644 --- a/src/ngx_http_json_log_module.c +++ b/src/ngx_http_json_log_module.c @@ -287,7 +287,19 @@ static ngx_int_t ngx_http_json_log_log_handler(ngx_http_request_t *r) if (ngx_json_log_write_sink_file(r->pool->log, location->file->fd, txt) == NGX_ERROR) { - ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, "Write Error!"); + ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, "File write error!"); + } + continue; + } + + /* Write to syslog */ + if (location->type == NGX_JSON_LOG_SINK_SYSLOG) { + if (!location->syslog) { + continue; + } + if (ngx_json_log_write_sink_syslog(r->pool->log, + r->pool, location->syslog, txt) == NGX_ERROR) { + ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, "Syslog write error!"); } continue; } @@ -472,6 +484,7 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) #if (NGX_HAVE_LIBRDKAFKA) && ! NGX_JSON_LOG_HAS_KAFKA_PREFIX(value) #endif + && ! NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value) ) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "Invalid prefix [%v] for HTTP log JSON output location", value); @@ -491,11 +504,13 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) new_location->type = NGX_JSON_LOG_SINK_FILE; prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; #if (NGX_HAVE_LIBRDKAFKA) - } - else if (NGX_JSON_LOG_HAS_KAFKA_PREFIX(value)) { + } else if (NGX_JSON_LOG_HAS_KAFKA_PREFIX(value)) { new_location->type = NGX_JSON_LOG_SINK_KAFKA; prefix_len = NGX_JSON_LOG_KAFKA_OUT_LEN; #endif + } else if (NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value)) { + new_location->type = NGX_JSON_LOG_SINK_SYSLOG; + prefix_len = NGX_JSON_LOG_SYSLOG_OUT_LEN; } /* Saves location without prefix. */ @@ -510,6 +525,18 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) &new_location->location); } + /* If sink type is syslog */ + if (new_location->type == NGX_JSON_LOG_SINK_SYSLOG) { + new_location->syslog = ngx_pcalloc(cf->pool, sizeof(ngx_syslog_peer_t)); + if (new_location->syslog == NULL) { + return NGX_CONF_ERROR; + } + + if (ngx_syslog_process_conf(cf, new_location->syslog) != NGX_CONF_OK) { + return NGX_CONF_ERROR; + } + } + #if (NGX_HAVE_LIBRDKAFKA) /* If sink type is kafka, then set topic config for this location */ if (new_location->type == NGX_JSON_LOG_SINK_KAFKA) { diff --git a/src/ngx_json_log_output.c b/src/ngx_json_log_output.c index 7043ea8..309b3ef 100644 --- a/src/ngx_json_log_output.c +++ b/src/ngx_json_log_output.c @@ -19,3 +19,43 @@ ngx_json_log_write_sink_file(ngx_log_t *log, } return NGX_OK; } + +ngx_int_t +ngx_json_log_write_sink_syslog(ngx_log_t *log, + ngx_pool_t* pool, + ngx_syslog_peer_t *syslog, + const char *txt) { + + ssize_t n = 0; + size_t size = 0; + u_char *line, *p; + size_t len = 0; + + len += sizeof("<255>Jan 01 00:00:00 ") - 1 + + ngx_cycle->hostname.len + 1 + + syslog->tag.len + 2 + + strlen(txt); + + line = ngx_pnalloc(pool, len); + if (line == NULL) { + return NGX_ERROR; + } + + p = ngx_syslog_add_header(syslog, line); + p = ngx_snprintf(p, len, "%s", txt); + size = p - line; + n = ngx_syslog_send(syslog, line, size); + + if (n < 0) { + ngx_log_error(NGX_LOG_WARN, log, 0, + "send() to syslog failed"); + return NGX_ERROR; + + } else if ((size_t) n != size) { + ngx_log_error(NGX_LOG_WARN, log, 0, + "send() to syslog has written only %z of %uz", + n, size); + return NGX_ERROR; + } + return NGX_OK; +} diff --git a/src/ngx_json_log_output.h b/src/ngx_json_log_output.h index 285cdb6..2f6ec68 100644 --- a/src/ngx_json_log_output.h +++ b/src/ngx_json_log_output.h @@ -44,10 +44,18 @@ NGX_JSON_LOG_KAFKA_OUT_LEN) == 0 ) #endif +#define NGX_JSON_LOG_SYSLOG_OUT_LEN (sizeof("syslog:") - 1) +#define NGX_JSON_LOG_HAS_SYSLOG_PREFIX(str) \ + (ngx_strncmp(str->data, \ + "syslog:", \ + NGX_JSON_LOG_SYSLOG_OUT_LEN) == 0 ) + + typedef enum { NGX_JSON_LOG_SINK_FILE = 0, + NGX_JSON_LOG_SINK_SYSLOG = 1, #if (NGX_HAVE_LIBRDKAFKA) - NGX_JSON_LOG_SINK_KAFKA = 1 + NGX_JSON_LOG_SINK_KAFKA = 2 #endif } ngx_json_log_sink_e; @@ -55,7 +63,8 @@ struct ngx_json_log_output_location_s { ngx_str_t location; ngx_json_log_sink_e type; ngx_json_log_format_t format; - ngx_open_file_t *file; + ngx_open_file_t *file; + ngx_syslog_peer_t *syslog; #if (NGX_HAVE_LIBRDKAFKA) ngx_json_log_kafka_conf_t kafka; #endif @@ -67,5 +76,11 @@ ngx_int_t ngx_json_log_write_sink_file(ngx_log_t *log, ngx_fd_t fd, const char *txt); +ngx_int_t +ngx_json_log_write_sink_syslog(ngx_log_t *log, + ngx_pool_t* pool, + ngx_syslog_peer_t *syslog, + const char *txt); + #endif // __NGX_JSON_LOG_OUTPUT_H__ diff --git a/src/ngx_stream_json_log_module.c b/src/ngx_stream_json_log_module.c index 7910b86..bf8037b 100644 --- a/src/ngx_stream_json_log_module.c +++ b/src/ngx_stream_json_log_module.c @@ -237,7 +237,19 @@ ngx_stream_json_log_log_handler(ngx_stream_session_t *s) if (ngx_json_log_write_sink_file(s->connection->pool->log, location->file->fd, txt) == NGX_ERROR) { ngx_log_error(NGX_LOG_EMERG, - s->connection->pool->log, 0, "Write Error!"); + s->connection->pool->log, 0, "File write error!"); + } + continue; + } + + /* Write to syslog */ + if (location->type == NGX_JSON_LOG_SINK_SYSLOG) { + if (!location->syslog) { + continue; + } + if (ngx_json_log_write_sink_syslog(s->connection->pool->log, + s->connection->pool,location->syslog, txt) == NGX_ERROR) { + ngx_log_error(NGX_LOG_EMERG, s->connection->pool->log, 0, "Syslog write error!"); } continue; } @@ -393,8 +405,12 @@ ngx_stream_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) value = &args[1]; - if (! NGX_JSON_LOG_HAS_FILE_PREFIX(value) && - ! NGX_JSON_LOG_HAS_KAFKA_PREFIX(value)) { + if (! NGX_JSON_LOG_HAS_FILE_PREFIX(value) +#if (NGX_HAVE_LIBRDKAFKA) + && ! NGX_JSON_LOG_HAS_KAFKA_PREFIX(value) +#endif + && ! NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value) + ) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "Invalid prefix [%v] for HTTP log JSON output location", value); return NGX_CONF_ERROR; @@ -418,6 +434,9 @@ ngx_stream_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) new_location->type = NGX_JSON_LOG_SINK_KAFKA; prefix_len = NGX_JSON_LOG_KAFKA_OUT_LEN; #endif + } else if (NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value)) { + new_location->type = NGX_JSON_LOG_SINK_SYSLOG; + prefix_len = NGX_JSON_LOG_SYSLOG_OUT_LEN; } /* Saves location without prefix. */ @@ -432,6 +451,18 @@ ngx_stream_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) &new_location->location); } + /* If sink type is syslog */ + if (new_location->type == NGX_JSON_LOG_SINK_SYSLOG) { + new_location->syslog = ngx_pcalloc(cf->pool, sizeof(ngx_syslog_peer_t)); + if (new_location->syslog == NULL) { + return NGX_CONF_ERROR; + } + + if (ngx_syslog_process_conf(cf, new_location->syslog) != NGX_CONF_OK) { + return NGX_CONF_ERROR; + } + } + #if (NGX_HAVE_LIBRDKAFKA) /* If sink type is kafka, then set topic config for this location */ if (new_location->type == NGX_JSON_LOG_SINK_KAFKA) { From eed8c9e59db128a187fa664a9e3c08150b58742e Mon Sep 17 00:00:00 2001 From: fooinha Date: 2017年12月23日 12:10:41 +0000 Subject: [PATCH 2/5] ngx-json-log: refactor output location --- src/ngx_http_json_log_filter_module.c | 63 ++------------ src/ngx_http_json_log_module.c | 82 ++---------------- src/ngx_json_log_output.c | 118 ++++++++++++++++++++++++++ src/ngx_json_log_output.h | 31 +++---- src/ngx_stream_json_log_module.c | 102 ++-------------------- 5 files changed, 145 insertions(+), 251 deletions(-) diff --git a/src/ngx_http_json_log_filter_module.c b/src/ngx_http_json_log_filter_module.c index 5aa3434..94d42e0 100644 --- a/src/ngx_http_json_log_filter_module.c +++ b/src/ngx_http_json_log_filter_module.c @@ -767,15 +767,10 @@ static char * ngx_http_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { ngx_http_json_log_srv_conf_t *lc = conf; - ngx_http_json_log_main_conf_t *mcf; - ngx_json_log_output_location_t *new_location = NULL; + ngx_http_json_log_main_conf_t *mcf = NULL; ngx_json_log_format_t *format; ngx_str_t *args = cf->args->elts; - ngx_str_t *value = NULL; - ngx_str_t *format_name; - ngx_uint_t found = 0; - size_t prefix_len; - size_t i; + ngx_json_log_output_location_t *new_location; if (! args) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, @@ -791,66 +786,20 @@ ngx_http_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - /* Check if format exists by name */ - format_name = &args[2]; - format = mcf->formats->elts; - for (i = 0; i < mcf->formats->nelts; i++) { - if (ngx_strncmp(format_name->data, format[i].name.data, - format[i].name.len) == 0) { - found = 1; - break; - } - } - + format = ngx_json_log_check_format(mcf->formats, &args[2]); /* Do not accept unknown format names */ - if (!found) { + if (format == NULL) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "http_json_log: Invalid format name [%V]", - format_name); - return NGX_CONF_ERROR; - } - - value = &args[1]; - - if (! NGX_JSON_LOG_HAS_FILE_PREFIX(value) -#if (NGX_HAVE_LIBRDKAFKA) - && ! NGX_JSON_LOG_HAS_KAFKA_PREFIX(value) -#endif - ) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "Invalid prefix [%v] for HTTP log JSON output location", value); + &args[2]); return NGX_CONF_ERROR; } - new_location = ngx_array_push(lc->locations); + new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, &args[1]); if (new_location == NULL) { return NGX_CONF_ERROR; } - ngx_memzero(new_location, sizeof(*new_location)); - prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; - if (NGX_JSON_LOG_HAS_FILE_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_FILE; - prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; -#if (NGX_HAVE_LIBRDKAFKA) - } - else if (NGX_JSON_LOG_HAS_KAFKA_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_KAFKA; - prefix_len = NGX_JSON_LOG_KAFKA_OUT_LEN; -#endif - } - - /* Saves location without prefix. */ - new_location->location = args[1]; - new_location->location.len -= prefix_len; - new_location->location.data += prefix_len; - new_location->format = format[i]; - - /* If sink type is file, then try to open it and save */ - if (new_location->type == NGX_JSON_LOG_SINK_FILE) { - new_location->file = ngx_conf_open_file(cf->cycle, - &new_location->location); - } #if (NGX_HAVE_LIBRDKAFKA) /* If sink type is kafka, then set topic config for this location */ diff --git a/src/ngx_http_json_log_module.c b/src/ngx_http_json_log_module.c index 2806aef..70a0002 100644 --- a/src/ngx_http_json_log_module.c +++ b/src/ngx_http_json_log_module.c @@ -437,14 +437,9 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { ngx_http_json_log_loc_conf_t *lc = conf; ngx_http_json_log_main_conf_t *mcf = NULL; - ngx_json_log_output_location_t *new_location = NULL; ngx_json_log_format_t *format; ngx_str_t *args = cf->args->elts; - ngx_str_t *value = NULL; - ngx_str_t *format_name; - ngx_uint_t found = 0; - size_t prefix_len; - size_t i; + ngx_json_log_output_location_t *new_location; if (! args) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, @@ -459,83 +454,19 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - /* Check if format exists by name */ - format_name = &args[2]; - format = mcf->formats->elts; - for (i = 0; i < mcf->formats->nelts; i++) { - if (ngx_strncmp(format_name->data, format[i].name.data, - format[i].name.len) == 0) { - found = 1; - break; - } - } - + format = ngx_json_log_check_format(mcf->formats, &args[2]); /* Do not accept unknown format names */ - if (!found) { + if (format == NULL) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "http_json_log: Invalid format name [%V]", - format_name); - return NGX_CONF_ERROR; - } - - value = &args[1]; - - if (! NGX_JSON_LOG_HAS_FILE_PREFIX(value) -#if (NGX_HAVE_LIBRDKAFKA) - && ! NGX_JSON_LOG_HAS_KAFKA_PREFIX(value) -#endif - && ! NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value) - ) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "Invalid prefix [%v] for HTTP log JSON output location", value); + &args[2]); return NGX_CONF_ERROR; } - new_location = ngx_array_push(lc->locations); - if (!new_location) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "Failed to add [%v] for HTTP log JSON output location", value); + new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, &args[1]); + if (new_location == NULL) { return NGX_CONF_ERROR; } - ngx_memzero(new_location, sizeof(*new_location)); - - prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; - if (NGX_JSON_LOG_HAS_FILE_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_FILE; - prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; -#if (NGX_HAVE_LIBRDKAFKA) - } else if (NGX_JSON_LOG_HAS_KAFKA_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_KAFKA; - prefix_len = NGX_JSON_LOG_KAFKA_OUT_LEN; -#endif - } else if (NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_SYSLOG; - prefix_len = NGX_JSON_LOG_SYSLOG_OUT_LEN; - } - - /* Saves location without prefix. */ - new_location->location = args[1]; - new_location->location.len -= prefix_len; - new_location->location.data += prefix_len; - new_location->format = format[i]; - - /* If sink type is file, then try to open it and save */ - if (new_location->type == NGX_JSON_LOG_SINK_FILE) { - new_location->file = ngx_conf_open_file(cf->cycle, - &new_location->location); - } - - /* If sink type is syslog */ - if (new_location->type == NGX_JSON_LOG_SINK_SYSLOG) { - new_location->syslog = ngx_pcalloc(cf->pool, sizeof(ngx_syslog_peer_t)); - if (new_location->syslog == NULL) { - return NGX_CONF_ERROR; - } - - if (ngx_syslog_process_conf(cf, new_location->syslog) != NGX_CONF_OK) { - return NGX_CONF_ERROR; - } - } #if (NGX_HAVE_LIBRDKAFKA) /* If sink type is kafka, then set topic config for this location */ @@ -562,7 +493,6 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) /* Set variable for message id */ ngx_memzero(&ccv, sizeof(ngx_http_compile_complex_value_t)); - ccv.cf = cf; ccv.value = &msg_id_variable; ccv.complex_value = ngx_pcalloc(cf->pool, diff --git a/src/ngx_json_log_output.c b/src/ngx_json_log_output.c index 309b3ef..1ee54a8 100644 --- a/src/ngx_json_log_output.c +++ b/src/ngx_json_log_output.c @@ -1,5 +1,123 @@ #include "ngx_json_log_output.h" +#include + + +#define NGX_JSON_LOG_FILE_OUT_LEN (sizeof("file:") - 1) +#define NGX_JSON_LOG_HAS_FILE_PREFIX(str) \ + (ngx_strncmp(str->data, \ + "file:", \ + NGX_JSON_LOG_FILE_OUT_LEN) == 0 ) + +#if (NGX_HAVE_LIBRDKAFKA) +#define NGX_JSON_LOG_KAFKA_OUT_LEN (sizeof("kafka:") - 1) +#define NGX_JSON_LOG_HAS_KAFKA_PREFIX(str) \ + (ngx_strncmp(str->data, \ + "kafka:", \ + NGX_JSON_LOG_KAFKA_OUT_LEN) == 0 ) +#endif + +#define NGX_JSON_LOG_SYSLOG_OUT_LEN (sizeof("syslog:") - 1) +#define NGX_JSON_LOG_HAS_SYSLOG_PREFIX(str) \ + (ngx_strncmp(str->data, \ + "syslog:", \ + NGX_JSON_LOG_SYSLOG_OUT_LEN) == 0 ) + + + +/* Check if format exists by name */ +ngx_json_log_format_t * +ngx_json_log_check_format(ngx_array_t *formats, + ngx_str_t *name) +{ + ngx_json_log_format_t *format; + size_t i; + + /* Check if format exists by name */ + format = formats->elts; + for (i = 0; i < formats->nelts; i++) { + if (ngx_strncmp(name->data, format[i].name.data, + format[i].name.len) == 0) { + return &format[i]; + } + } + + return NULL; +} + +ngx_json_log_output_location_t * +ngx_json_log_output_location_conf(ngx_conf_t *cf, + ngx_json_log_format_t *format, + ngx_array_t *locations, + ngx_str_t *value) +{ + ngx_json_log_output_location_t *new_location = NULL; + size_t prefix_len; + + if (! NGX_JSON_LOG_HAS_FILE_PREFIX(value) +#if (NGX_HAVE_LIBRDKAFKA) + && ! NGX_JSON_LOG_HAS_KAFKA_PREFIX(value) +#endif + && ! NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value) + ) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "Invalid prefix [%v] JSON log output location", value); + return NULL; + } + + new_location = ngx_array_push(locations); + if (!new_location) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "Failed to add [%v] JSON log output location", value); + return NULL; + } + ngx_memzero(new_location, sizeof(*new_location)); + + prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; + if (NGX_JSON_LOG_HAS_FILE_PREFIX(value)) { + new_location->type = NGX_JSON_LOG_SINK_FILE; + prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; +#if (NGX_HAVE_LIBRDKAFKA) + } else if (NGX_JSON_LOG_HAS_KAFKA_PREFIX(value)) { + new_location->type = NGX_JSON_LOG_SINK_KAFKA; + prefix_len = NGX_JSON_LOG_KAFKA_OUT_LEN; +#endif + } else if (NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value)) { + new_location->type = NGX_JSON_LOG_SINK_SYSLOG; + prefix_len = NGX_JSON_LOG_SYSLOG_OUT_LEN; + } + + /* Saves location without prefix. */ + new_location->location = *value; + new_location->location.len -= prefix_len; + new_location->location.data += prefix_len; + new_location->format = *format; + + /* If sink type is file, then try to open it and save */ + if (new_location->type == NGX_JSON_LOG_SINK_FILE) { + new_location->file = ngx_conf_open_file(cf->cycle, + &new_location->location); + } + + /* If sink type is syslog */ + if (new_location->type == NGX_JSON_LOG_SINK_SYSLOG) { + new_location->syslog = ngx_pcalloc(cf->pool, sizeof(ngx_syslog_peer_t)); + if (new_location->syslog == NULL) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "Failed to allocated JSON log output location to syslog"); + return NULL; + } + + if (ngx_syslog_process_conf(cf, new_location->syslog) != NGX_CONF_OK) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "Failed to process syslog JSON log output location"); + return NULL; + } + } + + return new_location; +} + ngx_int_t ngx_json_log_write_sink_file(ngx_log_t *log, ngx_fd_t fd, const char *txt) diff --git a/src/ngx_json_log_output.h b/src/ngx_json_log_output.h index 2f6ec68..65e563b 100644 --- a/src/ngx_json_log_output.h +++ b/src/ngx_json_log_output.h @@ -30,27 +30,6 @@ #include "ngx_json_log_text.h" #include "ngx_json_log_kafka.h" -#define NGX_JSON_LOG_FILE_OUT_LEN (sizeof("file:") - 1) -#define NGX_JSON_LOG_HAS_FILE_PREFIX(str) \ - (ngx_strncmp(str->data, \ - "file:", \ - NGX_JSON_LOG_FILE_OUT_LEN) == 0 ) - -#if (NGX_HAVE_LIBRDKAFKA) -#define NGX_JSON_LOG_KAFKA_OUT_LEN (sizeof("kafka:") - 1) -#define NGX_JSON_LOG_HAS_KAFKA_PREFIX(str) \ - (ngx_strncmp(str->data, \ - "kafka:", \ - NGX_JSON_LOG_KAFKA_OUT_LEN) == 0 ) -#endif - -#define NGX_JSON_LOG_SYSLOG_OUT_LEN (sizeof("syslog:") - 1) -#define NGX_JSON_LOG_HAS_SYSLOG_PREFIX(str) \ - (ngx_strncmp(str->data, \ - "syslog:", \ - NGX_JSON_LOG_SYSLOG_OUT_LEN) == 0 ) - - typedef enum { NGX_JSON_LOG_SINK_FILE = 0, NGX_JSON_LOG_SINK_SYSLOG = 1, @@ -72,6 +51,16 @@ struct ngx_json_log_output_location_s { typedef struct ngx_json_log_output_location_s ngx_json_log_output_location_t; +ngx_json_log_format_t * +ngx_json_log_check_format(ngx_array_t *formats, + ngx_str_t *name); + +ngx_json_log_output_location_t * +ngx_json_log_output_location_conf(ngx_conf_t *cf, + ngx_json_log_format_t *format, + ngx_array_t *locations, + ngx_str_t *value); + ngx_int_t ngx_json_log_write_sink_file(ngx_log_t *log, ngx_fd_t fd, const char *txt); diff --git a/src/ngx_stream_json_log_module.c b/src/ngx_stream_json_log_module.c index bf8037b..1f39478 100644 --- a/src/ngx_stream_json_log_module.c +++ b/src/ngx_stream_json_log_module.c @@ -364,11 +364,6 @@ ngx_stream_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) ngx_json_log_output_location_t *new_location = NULL; ngx_json_log_format_t *format; ngx_str_t *args = cf->args->elts; - ngx_str_t *value = NULL; - ngx_str_t *format_name; - ngx_uint_t found = 0; - size_t prefix_len; - size_t i; if (! args) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, @@ -383,85 +378,20 @@ ngx_stream_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - - /* Check if format exists by name */ - format_name = &args[2]; - format = mcf->formats->elts; - for (i = 0; i < mcf->formats->nelts; i++) { - if (ngx_strncmp(format_name->data, format[i].name.data, - format[i].name.len) == 0) { - found = 1; - break; - } - } - + format = ngx_json_log_check_format(mcf->formats, &args[2]); /* Do not accept unknown format names */ - if (!found) { + if (format == NULL) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "http_json_log: Invalid format name [%V]", - format_name); - return NGX_CONF_ERROR; - } - - value = &args[1]; - - if (! NGX_JSON_LOG_HAS_FILE_PREFIX(value) -#if (NGX_HAVE_LIBRDKAFKA) - && ! NGX_JSON_LOG_HAS_KAFKA_PREFIX(value) -#endif - && ! NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value) - ) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "Invalid prefix [%v] for HTTP log JSON output location", value); + &args[2]); return NGX_CONF_ERROR; } - new_location = ngx_array_push(lc->locations); - if (!new_location) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "Failed to add [%v] for HTTP log JSON output location", value); + new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, &args[1]); + if (new_location == NULL) { return NGX_CONF_ERROR; } - ngx_memzero(new_location, sizeof(*new_location)); - - prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; - if (NGX_JSON_LOG_HAS_FILE_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_FILE; - prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; -#if (NGX_HAVE_LIBRDKAFKA) - } - else if (NGX_JSON_LOG_HAS_KAFKA_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_KAFKA; - prefix_len = NGX_JSON_LOG_KAFKA_OUT_LEN; -#endif - } else if (NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_SYSLOG; - prefix_len = NGX_JSON_LOG_SYSLOG_OUT_LEN; - } - - /* Saves location without prefix. */ - new_location->location = args[1]; - new_location->location.len -= prefix_len; - new_location->location.data += prefix_len; - new_location->format = format[i]; - - /* If sink type is file, then try to open it and save */ - if (new_location->type == NGX_JSON_LOG_SINK_FILE) { - new_location->file = ngx_conf_open_file(cf->cycle, - &new_location->location); - } - /* If sink type is syslog */ - if (new_location->type == NGX_JSON_LOG_SINK_SYSLOG) { - new_location->syslog = ngx_pcalloc(cf->pool, sizeof(ngx_syslog_peer_t)); - if (new_location->syslog == NULL) { - return NGX_CONF_ERROR; - } - - if (ngx_syslog_process_conf(cf, new_location->syslog) != NGX_CONF_OK) { - return NGX_CONF_ERROR; - } - } #if (NGX_HAVE_LIBRDKAFKA) /* If sink type is kafka, then set topic config for this location */ @@ -480,28 +410,6 @@ ngx_stream_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) /* Set global variable */ stream_json_log_has_kafka_locations = NGX_OK; - -//FIXME: #if nginx_version>= 1011000 -//FIXME: ngx_http_compile_complex_value_t ccv; -//FIXME: /*FIXME: Change this to an user's configured variable */ -//FIXME: ngx_str_t msg_id_variable = ngx_string("$request_id"); -//FIXME: -//FIXME: /* Set variable for message id */ -//FIXME: ngx_memzero(&ccv, sizeof(ngx_stream_compile_complex_value_t)); -//FIXME: -//FIXME: -//FIXME: ccv.cf = cf; -//FIXME: ccv.value = &msg_id_variable; -//FIXME: ccv.complex_value = ngx_pcalloc(cf->pool, -//FIXME: sizeof(ngx_stream_complex_value_t)); -//FIXME: if (ccv.complex_value == NULL) { -//FIXME: return NGX_CONF_ERROR; -//FIXME: } -//FIXME: if (ngx_stream_compile_complex_value(&ccv) != NGX_OK) { -//FIXME: return NGX_CONF_ERROR; -//FIXME: } -//FIXME: new_location->kafka.stream_msg_id_var = ccv.complex_value; -//FIXME: #endif } #endif From 0ad2cb9c731364577b7018431f8df6b1307f9857 Mon Sep 17 00:00:00 2001 From: fooinha Date: 2017年12月23日 13:20:26 +0000 Subject: [PATCH 3/5] ngx-json-log: styling fixes --- docker/debian-nginx-http-log-json/vimrc | 4 ++-- src/ngx_http_json_log_filter_module.c | 29 ++++++++++++++----------- src/ngx_http_json_log_module.c | 9 +++++--- src/ngx_stream_json_log_module.c | 18 ++++++++------- 4 files changed, 34 insertions(+), 26 deletions(-) diff --git a/docker/debian-nginx-http-log-json/vimrc b/docker/debian-nginx-http-log-json/vimrc index cedc851..814843a 100644 --- a/docker/debian-nginx-http-log-json/vimrc +++ b/docker/debian-nginx-http-log-json/vimrc @@ -77,8 +77,8 @@ autocmd BufWinLeave * call clearmatches() " Source a global configuration file if available if filereadable("/etc/vim/vimrc.local") - source /etc/vim/vimrc.local - endif + source /etc/vim/vimrc.local +endif " CTags tag on cursor find definition map ? :tag =expand("") diff --git a/src/ngx_http_json_log_filter_module.c b/src/ngx_http_json_log_filter_module.c index 94d42e0..1d300b8 100644 --- a/src/ngx_http_json_log_filter_module.c +++ b/src/ngx_http_json_log_filter_module.c @@ -178,7 +178,8 @@ static ngx_command_t ngx_http_json_log_filter_commands[] = { NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, ngx_conf_set_num_slot, NGX_HTTP_MAIN_CONF_OFFSET, - offsetof(ngx_http_json_log_filter_main_conf_t, kafka.buffer_max_messages), + offsetof(ngx_http_json_log_filter_main_conf_t, + kafka.buffer_max_messages), NULL }, { @@ -195,14 +196,14 @@ static ngx_command_t ngx_http_json_log_filter_commands[] = { /* config preparation */ static ngx_http_module_t ngx_http_json_log_filter_module_ctx = { - NULL, /* preconfiguration */ - ngx_http_json_log_filter_init, /* postconfiguration */ - ngx_http_json_log_filter_create_main_conf, /* create main configuration */ - NULL, /* init main configuration */ - ngx_http_json_log_filter_create_srv_conf, /* create server configuration */ - NULL, /* merge server configuration */ - ngx_http_json_log_filter_create_loc_conf, /* create location configuration */ - NULL /* merge location configuration */ + NULL, /* preconfiguration */ + ngx_http_json_log_filter_init, /* postconfiguration */ + ngx_http_json_log_filter_create_main_conf,/* create main configuration */ + NULL, /* init main configuration */ + ngx_http_json_log_filter_create_srv_conf, /* create server configuration */ + NULL, /* merge server configuration */ + ngx_http_json_log_filter_create_loc_conf, /* create location configuration*/ + NULL /* merge location configuration */ }; ngx_module_t ngx_http_json_log_filter_module = { @@ -444,7 +445,8 @@ ngx_http_json_log_header_bad_request(ngx_http_request_t *r) } if (ngx_json_log_write_sink_syslog(r->pool->log, r->pool, location->syslog, txt) == NGX_ERROR) { - ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, "Syslog write error!"); + ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, + "Syslog write error!"); } continue; } @@ -673,7 +675,8 @@ ngx_http_json_log_body_filter(ngx_http_request_t *r, ngx_chain_t *in) if (!vv_hex) { return ngx_http_next_header_filter(r); } - ngx_http_json_log_set_variable_req_body_hexdump(r, vv_hex, (uintptr_t) &payload); + ngx_http_json_log_set_variable_req_body_hexdump(r, + vv_hex, (uintptr_t) &payload); return ngx_http_next_request_body_filter(r, in); } @@ -795,7 +798,8 @@ ngx_http_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, &args[1]); + new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, + &args[1]); if (new_location == NULL) { return NGX_CONF_ERROR; } @@ -864,7 +868,6 @@ ngx_http_json_log_filter_init_worker(ngx_cycle_t *cycle) return NGX_OK; } - rc = ngx_json_log_configure_kafka(cycle->pool, &conf->kafka); if (rc != NGX_OK) { return NGX_OK; //FIXME: What to do? diff --git a/src/ngx_http_json_log_module.c b/src/ngx_http_json_log_module.c index 70a0002..48cb59b 100644 --- a/src/ngx_http_json_log_module.c +++ b/src/ngx_http_json_log_module.c @@ -287,7 +287,8 @@ static ngx_int_t ngx_http_json_log_log_handler(ngx_http_request_t *r) if (ngx_json_log_write_sink_file(r->pool->log, location->file->fd, txt) == NGX_ERROR) { - ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, "File write error!"); + ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, + "File write error!"); } continue; } @@ -299,7 +300,8 @@ static ngx_int_t ngx_http_json_log_log_handler(ngx_http_request_t *r) } if (ngx_json_log_write_sink_syslog(r->pool->log, r->pool, location->syslog, txt) == NGX_ERROR) { - ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, "Syslog write error!"); + ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, + "Syslog write error!"); } continue; } @@ -463,7 +465,8 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, &args[1]); + new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, + &args[1]); if (new_location == NULL) { return NGX_CONF_ERROR; } diff --git a/src/ngx_stream_json_log_module.c b/src/ngx_stream_json_log_module.c index 1f39478..6149982 100644 --- a/src/ngx_stream_json_log_module.c +++ b/src/ngx_stream_json_log_module.c @@ -206,7 +206,8 @@ ngx_stream_json_log_log_handler(ngx_stream_session_t *s) /* Check filter result */ if (location->format.stream_filter != NULL) { if (ngx_stream_complex_value(s, - location->format.stream_filter, &filter_val) != NGX_OK) { + location->format.stream_filter, + &filter_val) != NGX_OK) { /* WARN ? */ continue; } @@ -248,8 +249,10 @@ ngx_stream_json_log_log_handler(ngx_stream_session_t *s) continue; } if (ngx_json_log_write_sink_syslog(s->connection->pool->log, - s->connection->pool,location->syslog, txt) == NGX_ERROR) { - ngx_log_error(NGX_LOG_EMERG, s->connection->pool->log, 0, "Syslog write error!"); + s->connection->pool, + location->syslog, txt) == NGX_ERROR) { + ngx_log_error(NGX_LOG_EMERG, s->connection->pool->log, 0, + "Syslog write error!"); } continue; } @@ -343,8 +346,6 @@ ngx_stream_json_log_post_conf(ngx_conf_t *cf) cmcf = ngx_stream_conf_get_module_main_conf(cf, ngx_stream_core_module); - //TODO-OPTIMIZATION: to verify if the module should be registered on log phase - h = ngx_array_push(&cmcf->phases[NGX_STREAM_LOG_PHASE].handlers); if (h == NULL) { return NGX_ERROR; @@ -387,12 +388,12 @@ ngx_stream_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, &args[1]); + new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, + &args[1]); if (new_location == NULL) { return NGX_CONF_ERROR; } - #if (NGX_HAVE_LIBRDKAFKA) /* If sink type is kafka, then set topic config for this location */ if (new_location->type == NGX_JSON_LOG_SINK_KAFKA) { @@ -423,7 +424,8 @@ ngx_stream_json_log_init_worker(ngx_cycle_t *cycle) { #if (NGX_HAVE_LIBRDKAFKA) ngx_stream_json_log_main_conf_t *conf = - ngx_stream_cycle_get_module_main_conf(cycle, ngx_stream_json_log_module); + ngx_stream_cycle_get_module_main_conf(cycle, + ngx_stream_json_log_module); /* from this point we just are init kafka stuff */ if (stream_json_log_has_kafka_locations == NGX_CONF_UNSET ) { From 4f4ad79f30a9eb81af4cacf3d1daf50eb7a00428 Mon Sep 17 00:00:00 2001 From: fooinha Date: 2017年11月27日 11:02:57 +0000 Subject: [PATCH 4/5] ngx-json-log: code cleanup --- src/ngx_http_json_log_filter_module.c | 35 ++++++++-------- src/ngx_http_json_log_module.c | 21 ++++------ src/ngx_http_json_log_module.h | 2 +- src/ngx_http_json_log_variables.c | 18 ++------ src/ngx_json_log_kafka.h | 8 ++-- src/ngx_json_log_output.c | 5 ++- src/ngx_json_log_str.c | 22 +++------- src/ngx_json_log_text.c | 32 +++++++-------- src/ngx_stream_json_log_module.c | 6 +-- src/ngx_stream_json_log_preread_module.c | 52 +----------------------- 10 files changed, 62 insertions(+), 139 deletions(-) diff --git a/src/ngx_http_json_log_filter_module.c b/src/ngx_http_json_log_filter_module.c index 1d300b8..ac8323b 100644 --- a/src/ngx_http_json_log_filter_module.c +++ b/src/ngx_http_json_log_filter_module.c @@ -26,7 +26,6 @@ #include #include #include -#include #include "ngx_http_json_log_module.h" #include "ngx_http_json_log_variables.h" @@ -274,7 +273,7 @@ ngx_http_json_log_loc_req_body_limit(ngx_conf_t *cf, { ngx_http_json_log_filter_loc_conf_t *lc = conf; ngx_str_t *args = cf->args->elts; - size_t sp = NGX_ERROR; + ssize_t sp; if (! args) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, @@ -283,7 +282,7 @@ ngx_http_json_log_loc_req_body_limit(ngx_conf_t *cf, } sp = ngx_parse_size(&args[1]); - if (sp == (size_t) NGX_ERROR) { + if (sp == NGX_ERROR) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "Invalid argument for HTTP request body limit"); return NGX_CONF_ERROR; @@ -295,11 +294,9 @@ ngx_http_json_log_loc_req_body_limit(ngx_conf_t *cf, lc->req_body_limit = sp; - return NGX_CONF_OK; } - static size_t ngx_http_json_log_get_req_body_limit(ngx_http_request_t *r) { @@ -313,31 +310,31 @@ ngx_http_json_log_get_req_body_limit(ngx_http_request_t *r) return lc->req_body_limit; } - static void ngx_http_json_log_header_bad_request(ngx_http_request_t *r) { ngx_http_json_log_srv_conf_t *lc; - ngx_str_t filter_val; + ngx_str_t filter_val; char *txt; - size_t i, len; + size_t i; + size_t len; ngx_json_log_output_location_t *arr; ngx_json_log_output_location_t *location; - size_t limit = HTTP_LOG_JSON_REQ_BODY_LIMIT_DEFAULT; - ngx_str_t name = ngx_string("http_json_err_log_req"); - ngx_str_t lcname; - ngx_http_variable_value_t *vv; - ngx_uint_t varkey; + size_t limit; + ngx_str_t name = ngx_string("http_json_err_log_req"); + ngx_str_t lcname; + ngx_http_variable_value_t *vv; + ngx_uint_t varkey; - ngx_str_t name_hex = ngx_string("http_json_err_log_req_hexdump"); - ngx_str_t lcname_hex; - ngx_http_variable_value_t *vv_hex; - ngx_uint_t varkey_hex; + ngx_str_t name_hex = ngx_string("http_json_err_log_req_hexdump"); + ngx_str_t lcname_hex; + ngx_http_variable_value_t *vv_hex; + ngx_uint_t varkey_hex; - ngx_str_t payload; + ngx_str_t payload; #if (NGX_HAVE_LIBRDKAFKA) - ngx_str_t msg_id; + ngx_str_t msg_id; ngx_http_json_log_filter_main_conf_t *mcf; #endif diff --git a/src/ngx_http_json_log_module.c b/src/ngx_http_json_log_module.c index 48cb59b..f66cc33 100644 --- a/src/ngx_http_json_log_module.c +++ b/src/ngx_http_json_log_module.c @@ -28,19 +28,11 @@ #include #include -#include -#include - #include "ngx_http_json_log_module.h" -#include "ngx_json_log_str.h" -#include "ngx_json_log_output.h" -#include "ngx_json_log_text.h" #include "ngx_http_json_log_variables.h" +#include "ngx_json_log_output.h" #if (NGX_HAVE_LIBRDKAFKA) - -#include "ngx_json_log_kafka.h" - /*Global variable to indicate the we have kafka locations*/ static ngx_int_t http_json_log_has_kafka_locations = NGX_CONF_UNSET; /* configuration kafka constants */ @@ -51,6 +43,10 @@ static char * ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +static ngx_str_t msg_id_variable = ngx_string("$request_id"); + + + static void * ngx_http_json_log_create_main_conf(ngx_conf_t *cf); static void * ngx_http_json_log_create_loc_conf(ngx_conf_t *cf); @@ -183,7 +179,7 @@ static ngx_int_t ngx_http_json_log_init_worker(ngx_cycle_t *cycle) { #if (NGX_HAVE_LIBRDKAFKA) - ngx_int_t rc = NGX_OK; + ngx_int_t rc; ngx_http_json_log_main_conf_t *conf = ngx_http_cycle_get_module_main_conf(cycle, ngx_http_json_log_module); @@ -213,9 +209,9 @@ ngx_http_json_log_exit_worker(ngx_cycle_t *cycle) /* log handler - format and print */ static ngx_int_t ngx_http_json_log_log_handler(ngx_http_request_t *r) { - ngx_http_json_log_loc_conf_t *lc; + ngx_http_json_log_loc_conf_t *lc; ngx_str_t filter_val; - char *txt; + char *txt; size_t i; ngx_json_log_output_location_t *arr; ngx_json_log_output_location_t *location; @@ -491,7 +487,6 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) #if nginx_version>= 1011000 ngx_http_compile_complex_value_t ccv; /*FIXME: Change this to an user's configured variable */ - ngx_str_t msg_id_variable = ngx_string("$request_id"); /* Set variable for message id */ ngx_memzero(&ccv, sizeof(ngx_http_compile_complex_value_t)); diff --git a/src/ngx_http_json_log_module.h b/src/ngx_http_json_log_module.h index 3cc90b0..27aa205 100644 --- a/src/ngx_http_json_log_module.h +++ b/src/ngx_http_json_log_module.h @@ -26,7 +26,7 @@ #ifndef __NGX_HTTP_JSON_LOG_MODULE_H__ #define __NGX_HTTP_JSON_LOG_MODULE_H__ -#define NGX_JSON_LOG_VER "0.0.5" +#define NGX_JSON_LOG_VER "0.0.7" #include diff --git a/src/ngx_http_json_log_variables.c b/src/ngx_http_json_log_variables.c index 88ad4da..a2c9a43 100644 --- a/src/ngx_http_json_log_variables.c +++ b/src/ngx_http_json_log_variables.c @@ -25,11 +25,6 @@ */ #include #include -#include -#include -#include -#include -#include #include "ngx_http_json_log_variables.h" #include "ngx_json_log_text.h" @@ -37,12 +32,6 @@ #include -#include -#include -#include -#include -#include - typedef ngx_queue_t *(*get_body_queue_pt)(ngx_http_request_t *r); static ngx_int_t @@ -264,10 +253,9 @@ ngx_http_json_log_set_variable_resp_headers(ngx_http_request_t *r, #endif } - if (object) { - v->valid = 1; - v->data = (void *) object; - } + v->valid = 1; + v->data = (void *) object; + set_current_mem_pool(NULL); } diff --git a/src/ngx_json_log_kafka.h b/src/ngx_json_log_kafka.h index e1c5e00..c354dbf 100644 --- a/src/ngx_json_log_kafka.h +++ b/src/ngx_json_log_kafka.h @@ -49,9 +49,9 @@ typedef struct ngx_json_log_kafka_conf_s ngx_json_log_kafka_conf_t; /* configuration data structures */ struct ngx_json_log_main_kafka_conf_s { - rd_kafka_t *rk; /* kafka connection handler */ - rd_kafka_conf_t *rkc; /* kafka configuration */ - ngx_array_t *brokers; /* kafka list of brokers */ + rd_kafka_t *rk; /* kafka connection handler */ + rd_kafka_conf_t *rkc; /* kafka configuration */ + ngx_array_t *brokers; /* kafka list of brokers */ size_t valid_brokers; /* number of valid brokers added */ ngx_str_t client_id; /* kafka client id */ ngx_str_t compression; /* kafka communication compression */ @@ -63,7 +63,7 @@ struct ngx_json_log_main_kafka_conf_s { }; typedef struct ngx_json_log_main_kafka_conf_s ngx_json_log_main_kafka_conf_t; -/* topic confifuration */ +/* topic configuration */ rd_kafka_topic_conf_t * ngx_json_log_kafka_topic_conf_new(ngx_pool_t* pool); diff --git a/src/ngx_json_log_output.c b/src/ngx_json_log_output.c index 1ee54a8..77047e2 100644 --- a/src/ngx_json_log_output.c +++ b/src/ngx_json_log_output.c @@ -122,13 +122,14 @@ ngx_int_t ngx_json_log_write_sink_file(ngx_log_t *log, ngx_fd_t fd, const char *txt) { - size_t written = 0, len = 0; + ssize_t written = 0; + ssize_t len = 0; if (!txt) { return NGX_ERROR; } len = strlen(txt); - written = ngx_write_fd(fd, (u_char *)txt, len); + written = ngx_write_fd(fd, (u_char *)txt, (size_t) len); if (len && len != written) { ngx_log_error(NGX_LOG_EMERG, log, 0, "mismatch size: fd=%d len=%d written=%d", diff --git a/src/ngx_json_log_str.c b/src/ngx_json_log_str.c index 1e3a8ce..ad1a6ec 100644 --- a/src/ngx_json_log_str.c +++ b/src/ngx_json_log_str.c @@ -23,7 +23,8 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ -#include + +#include /* duplicates and set as null terminated */ @@ -46,7 +47,9 @@ ngx_str_t * ngx_json_log_str_dup_from_buf_len(ngx_pool_t *pool, ngx_str_t *src, size_t len) { - ngx_str_t *str = ngx_pcalloc(pool, sizeof(ngx_str_t)); + ngx_str_t *str; + + str = ngx_pcalloc(pool, sizeof(ngx_str_t)); if (str == NULL) { return NULL; } @@ -66,7 +69,7 @@ ngx_json_log_str_dup_from_buf_len(ngx_pool_t *pool, const char * ngx_json_log_buf_dup_len(ngx_pool_t *pool, u_char *src, size_t len) { - char *dst; + char *dst; dst = ngx_pcalloc(pool, len + 1); if (dst == NULL) { @@ -189,19 +192,6 @@ ngx_json_log_hexdump(ngx_str_t *src, ngx_str_t *dst) blocks = dst->len / blocksz; pos = dst->data; for (b = 0; b <= blocks ; ++b) { -/* - if (b * blocksz < src->len) { - snprintf((char *)pos, 8, - "%08X", (unsigned int) (b * blocksz)); - - pos += 8; - ngx_snprintf(pos, 2, " "); - pos += 2; - //printf("%08x ", (unsigned int) (b * blocksz)); - } else { - break; - } -*/ for (i = 0; i < blocksz ; ++i) { l = (b * blocksz) + i; diff --git a/src/ngx_json_log_text.c b/src/ngx_json_log_text.c index 36c125c..1e96f58 100644 --- a/src/ngx_json_log_text.c +++ b/src/ngx_json_log_text.c @@ -25,7 +25,6 @@ */ #include #include -#include #include #include "ngx_json_log_text.h" @@ -52,8 +51,8 @@ struct ngx_json_log_output_cxt_s { /* array to keep levels node values */ /* no need for hash or list struct */ /* as it should be very small */ - json_t * root; - ngx_array_t * items; + json_t *root; + ngx_array_t *items; }; @@ -177,10 +176,10 @@ ngx_json_log_find_saved_parent(ngx_pool_t *pool, static const char * ngx_json_log_label_key_dup(ngx_pool_t *pool, ngx_str_t *path, size_t max) { - u_char *copy = NULL; - int l = ngx_min(path->len, max); - int start= l - 1; - int i; + u_char *copy; + int l = ngx_min(path->len, max); + int start= l - 1; + int i; if (!path || !path->data || !path->len) return NULL; @@ -330,7 +329,7 @@ static void ngx_json_log_add_json_node( json_t *base, u_char *nptr = ngx_json_log_str_dup(pool, value); if (nptr) { char *endptr = (char *) nptr + value->len; - double val_real = strtold((const char *)nptr, &endptr); + double val_real = strtod((const char *)nptr, &endptr); node = json_real(val_real); } } else { @@ -357,14 +356,14 @@ ngx_json_log_output_add_item( ngx_json_log_output_cxt_t *output_ctx, ngx_json_log_item_t *item) { - ngx_str_t value; - uint32_t levels = 0; + ngx_str_t value; + uint32_t levels = 0; json_t *parent = output_ctx->root; - ngx_str_t lcname; - ngx_uint_t varkey; + ngx_str_t lcname; + ngx_uint_t varkey; ngx_http_variable_value_t *vv; - const char *key = NULL; + const char *key = NULL; ngx_http_complex_value_t *http_ccv = NULL; @@ -373,7 +372,7 @@ ngx_json_log_output_add_item( ngx_stream_complex_value_t *stream_ccv = NULL; #endif - ngx_int_t err = 0; + ngx_int_t err = 0; ngx_http_request_t *r = NULL; if (type == NGX_JSON_LOG_HTTP) { @@ -444,9 +443,10 @@ char * ngx_json_log_items_dump_text(ngx_json_log_module_type_e type, void *rs, ngx_array_t *items) { - ngx_json_log_output_cxt_t ctx; + ngx_json_log_output_cxt_t ctx; ngx_json_log_item_t *item; - size_t i, dump_len; + size_t i; + size_t dump_len; char *txt = NULL; char *dump = NULL; ngx_http_request_t *r = NULL; diff --git a/src/ngx_stream_json_log_module.c b/src/ngx_stream_json_log_module.c index 6149982..b797a68 100644 --- a/src/ngx_stream_json_log_module.c +++ b/src/ngx_stream_json_log_module.c @@ -170,15 +170,15 @@ ngx_module_t ngx_stream_json_log_module = { static ngx_int_t ngx_stream_json_log_log_handler(ngx_stream_session_t *s) { - ngx_stream_json_log_srv_conf_t *lc; + ngx_stream_json_log_srv_conf_t *lc; ngx_str_t filter_val; - char *txt; + char *txt; size_t i; ngx_json_log_output_location_t *arr; ngx_json_log_output_location_t *location; #if (NGX_HAVE_LIBRDKAFKA) - ngx_stream_json_log_main_conf_t *mcf; + ngx_stream_json_log_main_conf_t *mcf; mcf = ngx_stream_get_module_main_conf(s, ngx_stream_json_log_module); #endif diff --git a/src/ngx_stream_json_log_preread_module.c b/src/ngx_stream_json_log_preread_module.c index 032ec58..5c89b5d 100644 --- a/src/ngx_stream_json_log_preread_module.c +++ b/src/ngx_stream_json_log_preread_module.c @@ -27,8 +27,6 @@ #include #include -#include - typedef struct { ngx_flag_t enabled; } ngx_stream_json_log_preread_srv_conf_t; @@ -60,8 +58,8 @@ static ngx_stream_module_t ngx_stream_json_log_preread_module_ctx = { NULL, /* create main configuration */ NULL, /* init main configuration */ - ngx_stream_json_log_preread_create_srv_conf, /* create server configuration */ - ngx_stream_json_log_preread_merge_srv_conf /* merge server configuration */ + ngx_stream_json_log_preread_create_srv_conf,/* create server configuration*/ + ngx_stream_json_log_preread_merge_srv_conf /* merge server configuration */ }; @@ -87,16 +85,9 @@ ngx_stream_json_log_preread_handler(ngx_stream_session_t *s) ngx_connection_t *c; ngx_stream_json_log_preread_ctx_t *ctx; -// ngx_stream_json_log_preread_srv_conf_t *sscf; c = s->connection; -// sscf = ngx_stream_get_module_srv_conf(s, ngx_stream_json_log_preread_module); - -// if (!sscf->enabled) { -// return NGX_DECLINED; -// } - if (c->type != SOCK_STREAM) { return NGX_DECLINED; } @@ -121,9 +112,6 @@ ngx_stream_json_log_preread_handler(ngx_stream_session_t *s) ctx->last = c->buffer->last; - //printf("H P>%p L>%p S>%lu\n" , - // ctx->pos, ctx->last, ctx->last-ctx->pos); - return NGX_OK; } @@ -218,52 +206,16 @@ ngx_stream_json_log_preread_payload_variable(ngx_stream_session_t *s, } -//FIXME -//static ngx_int_t -//ngx_stream_json_log_preread_payload_hex_variable(ngx_stream_session_t *s, -// ngx_variable_value_t *v, uintptr_t data) -//{ -// ngx_stream_json_log_preread_ctx_t *ctx; -// size_t len; -// ngx_str_t payload; -// -// ctx = ngx_stream_get_module_ctx(s, ngx_stream_json_log_preread_module); -// -// if (ctx == NULL) { -// v->not_found = 1; -// return NGX_OK; -// } -// -// len = ctx->last-ctx->pos; -// -// if (ctx->payload.data == NULL && len) { -// //FIXME -// } -// -// v->valid = 1; -// v->no_cacheable = 0; -// v->not_found = 0; -// -// return NGX_OK; -//} - - static ngx_int_t ngx_stream_json_log_preread_add_variables(ngx_conf_t *cf) { ngx_stream_variable_t *var; ngx_str_t payload = ngx_string("ngx_stream_json_log_payload"); -// ngx_stream_variable_t *var_hex; -// ngx_str_t payload_hex = -// ngx_string("ngx_stream_json_log_payload_hexdump"); var = ngx_stream_add_variable(cf, &payload, 0); var->get_handler = ngx_stream_json_log_preread_payload_variable; -// var_hex = ngx_stream_add_variable(cf, &payload_hex, 0); -// var_hex->get_handler = ngx_stream_json_log_preread_payload_hex_variable; - return NGX_OK; } From b096355c57a18d3251341b48fc5436de7ab1b08f Mon Sep 17 00:00:00 2001 From: fooinha Date: 2017年12月23日 14:01:46 +0000 Subject: [PATCH 5/5] ngx-json-log: experimental upstream --- src/ngx_http_json_log_module.c | 86 ++++++++++++++++++++++++++++++++-- src/ngx_json_log_output.c | 55 ++++++++++++++++++++++ src/ngx_json_log_output.h | 29 +++++++++++- src/ngx_json_log_text.h | 12 +++-- 4 files changed, 172 insertions(+), 10 deletions(-) diff --git a/src/ngx_http_json_log_module.c b/src/ngx_http_json_log_module.c index f66cc33..1304522 100644 --- a/src/ngx_http_json_log_module.c +++ b/src/ngx_http_json_log_module.c @@ -26,7 +26,6 @@ #include #include #include -#include #include "ngx_http_json_log_module.h" #include "ngx_http_json_log_variables.h" @@ -344,6 +343,81 @@ static ngx_int_t ngx_http_json_log_log_handler(ngx_http_request_t *r) } // if KAFKA type #endif + + if (location->type == NGX_JSON_LOG_SINK_UPSTREAM) { + + if (!location->upstream) { + continue; + } +/* + ngx_http_upstream_t *u; + + u = ngx_pcalloc(r->pool, sizeof(ngx_http_upstream_t)); + if (u == NULL) { + continue; + } + + u-> + + ngx_int_t rc = ngx_event_connect_peer(location->upstream); + + ngx_log_error(NGX_LOG_INFO, r->pool->log, 0, + "http_json_log: u-conn[%lu]\n", + rc + ); +*/ + + //TODO : LOCKED TO FIRST ELEMENT AND FIRST ADDRESS + ngx_http_upstream_server_t *arr = location->upstream->servers->elts; + ngx_http_upstream_server_t *server = &arr[0]; + + ngx_log_error(NGX_LOG_INFO, r->pool->log, 0, + "http_json_log: name:[%v] down:[%ul] naddrs=[%ul]\n", + &server->name, server->down, server->naddrs + ); + + ngx_addr_t *addrs = server->addrs; + ngx_socket_t s = ngx_socket(addrs->sockaddr->sa_family, + SOCK_STREAM, 0); + ngx_connection_t *c = ngx_get_connection(s, r->pool->log); + + if (c == NULL) { + if (ngx_close_socket(s) == -1) { + ngx_log_error(NGX_LOG_ALERT, r->pool->log, ngx_socket_errno, + ngx_close_socket_n + "failed"); + } + continue; + } + + c->recv = ngx_recv; + c->send = ngx_send; + c->recv_chain = ngx_recv_chain; + c->send_chain = ngx_send_chain; + + int rc = connect(s, addrs->sockaddr, addrs->socklen); + if (rc == -1) { + ngx_close_connection(c); + continue; + } + + ngx_http_request_t *req = ngx_http_create_request(c); + + if (req == NULL) { + continue; + } + + req->keepalive = 1; + + //write(s, "HEAD / HTTP/1.0\r\n\r\n", 19); + //printf("XXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n"); + + //ngx_close_connection(c); + continue; + + + } // for upstream + } // for location return NGX_OK; @@ -427,8 +501,9 @@ ngx_http_json_log_create_loc_conf(ngx_conf_t *cf) * * Supported output destinations: * - * file: -> filesystem - * kafka: -> kafka topic + * file: -> filesystem + * kafka: -> kafka topic + * upstream: -> upstream connection */ static char * ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) @@ -439,6 +514,9 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) ngx_str_t *args = cf->args->elts; ngx_json_log_output_location_t *new_location; + + + if (! args) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "Invalid argument for HTTP log JSON output location"); @@ -467,8 +545,8 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } -#if (NGX_HAVE_LIBRDKAFKA) /* If sink type is kafka, then set topic config for this location */ +#if (NGX_HAVE_LIBRDKAFKA) if (new_location->type == NGX_JSON_LOG_SINK_KAFKA) { /* create topic conf */ new_location->kafka.rktc = ngx_json_log_kafka_topic_conf_new(cf->pool); diff --git a/src/ngx_json_log_output.c b/src/ngx_json_log_output.c index 77047e2..1dbb571 100644 --- a/src/ngx_json_log_output.c +++ b/src/ngx_json_log_output.c @@ -54,11 +54,17 @@ ngx_json_log_output_location_conf(ngx_conf_t *cf, ngx_json_log_output_location_t *new_location = NULL; size_t prefix_len; + size_t add; + u_short port; + ngx_url_t u; + ngx_str_t *url = NULL; + if (! NGX_JSON_LOG_HAS_FILE_PREFIX(value) #if (NGX_HAVE_LIBRDKAFKA) && ! NGX_JSON_LOG_HAS_KAFKA_PREFIX(value) #endif && ! NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value) + && ! NGX_JSON_LOG_HAS_UPSTREAM_PREFIX(value) ) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "Invalid prefix [%v] JSON log output location", value); @@ -85,6 +91,9 @@ ngx_json_log_output_location_conf(ngx_conf_t *cf, } else if (NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value)) { new_location->type = NGX_JSON_LOG_SINK_SYSLOG; prefix_len = NGX_JSON_LOG_SYSLOG_OUT_LEN; + } else if (NGX_JSON_LOG_HAS_UPSTREAM_PREFIX(value)) { + new_location->type = NGX_JSON_LOG_SINK_UPSTREAM; + prefix_len = NGX_JSON_LOG_UPSTREAM_OUT_LEN; } /* Saves location without prefix. */ @@ -115,6 +124,52 @@ ngx_json_log_output_location_conf(ngx_conf_t *cf, } } + /* If sink type is upstream, then validate upstream */ + if (new_location->type == NGX_JSON_LOG_SINK_UPSTREAM) { + + url = &new_location->location; + + //TODO: to work with a script location + //ngx_uint_t n = ngx_http_script_variables_count(&new_location->location); + + if (ngx_strncasecmp(url->data, (u_char *) "http://", 7) == 0) { + add = 7; + port = 80; + + } else if (ngx_strncasecmp(url->data, (u_char *) "https://", 8) == 0) { + +#if (NGX_HTTP_SSL) + plcf->ssl = 1; + + add = 8; + port = 443; +#else + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "https protocol requires SSL support"); + return NGX_CONF_ERROR; +#endif + + } else { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "invalid URL prefix"); + return NGX_CONF_ERROR; + } + + ngx_memzero(&u, sizeof(ngx_url_t)); + + u.url.len = url->len - add; + u.url.data = url->data + add; + u.default_port = port; + u.uri_part = 1; + u.no_resolve = 1; + + new_location->upstream = ngx_http_upstream_add(cf, &u, 0); + + if (new_location->upstream) { + printf("UPSTREAM %lu\n", new_location->upstream->servers->nelts); + } + + } + return new_location; } diff --git a/src/ngx_json_log_output.h b/src/ngx_json_log_output.h index 65e563b..651f23e 100644 --- a/src/ngx_json_log_output.h +++ b/src/ngx_json_log_output.h @@ -30,12 +30,33 @@ #include "ngx_json_log_text.h" #include "ngx_json_log_kafka.h" +#define NGX_JSON_LOG_FILE_OUT_LEN (sizeof("file:") - 1) +#define NGX_JSON_LOG_HAS_FILE_PREFIX(str) \ + (ngx_strncmp(str->data, \ + "file:", \ + NGX_JSON_LOG_FILE_OUT_LEN) == 0 ) + +#if (NGX_HAVE_LIBRDKAFKA) +#define NGX_JSON_LOG_KAFKA_OUT_LEN (sizeof("kafka:") - 1) +#define NGX_JSON_LOG_HAS_KAFKA_PREFIX(str) \ + (ngx_strncmp(str->data, \ + "kafka:", \ + NGX_JSON_LOG_KAFKA_OUT_LEN) == 0 ) +#endif + +#define NGX_JSON_LOG_UPSTREAM_OUT_LEN (sizeof("upstream:") - 1) +#define NGX_JSON_LOG_HAS_UPSTREAM_PREFIX(str) \ + (ngx_strncmp(str->data, \ + "upstream:", \ + NGX_JSON_LOG_UPSTREAM_OUT_LEN) == 0 ) + typedef enum { NGX_JSON_LOG_SINK_FILE = 0, NGX_JSON_LOG_SINK_SYSLOG = 1, #if (NGX_HAVE_LIBRDKAFKA) - NGX_JSON_LOG_SINK_KAFKA = 2 + NGX_JSON_LOG_SINK_KAFKA = 2, #endif + NGX_JSON_LOG_SINK_UPSTREAM = 3 } ngx_json_log_sink_e; struct ngx_json_log_output_location_s { @@ -47,6 +68,7 @@ struct ngx_json_log_output_location_s { #if (NGX_HAVE_LIBRDKAFKA) ngx_json_log_kafka_conf_t kafka; #endif + ngx_http_upstream_srv_conf_t *upstream; }; typedef struct ngx_json_log_output_location_s ngx_json_log_output_location_t; @@ -71,5 +93,10 @@ ngx_json_log_write_sink_syslog(ngx_log_t *log, ngx_syslog_peer_t *syslog, const char *txt); +ngx_int_t +ngx_json_log_write_sink_upstream(ngx_log_t *log, + ngx_fd_t fd, const char *txt); + + #endif // __NGX_JSON_LOG_OUTPUT_H__ diff --git a/src/ngx_json_log_text.h b/src/ngx_json_log_text.h index 306bd21..076341d 100644 --- a/src/ngx_json_log_text.h +++ b/src/ngx_json_log_text.h @@ -40,12 +40,12 @@ typedef enum { } ngx_json_log_module_type_e; struct ngx_json_log_format_s { - ngx_str_t name; /* the format name */ - ngx_str_t config; /* value at config files */ - ngx_array_t *items; /* format items */ - ngx_http_complex_value_t *http_filter; /* filter output */ + ngx_str_t name; /* the format name */ + ngx_str_t config; /* value at config files */ + ngx_array_t *items; /* format items */ + ngx_http_complex_value_t *http_filter; /* filter output */ #if nginx_version>= 1011002 - ngx_stream_complex_value_t *stream_filter; /* filter output */ + ngx_stream_complex_value_t *stream_filter; /* filter output */ #endif }; typedef struct ngx_json_log_format_s ngx_json_log_format_t; @@ -84,5 +84,7 @@ ngx_json_log_items_dump_text(ngx_json_log_module_type_e type, ngx_int_t ngx_json_log_items_cmp(const void *left, const void *right); + + #endif // __NGX_JSON_LOG_TEXT_H__

AltStyle によって変換されたページ (->オリジナル) /