From 32beb2bf5d6f51d46f3ed52995f04a72b1284c0f Mon Sep 17 00:00:00 2001 From: fooinha Date: 2017年12月21日 11:58:51 +0000 Subject: [PATCH 1/3] ngx-json-log: log to syslog --- README.md | 5 ++-- src/ngx_http_json_log_filter_module.c | 12 ++++++++ src/ngx_http_json_log_module.c | 33 ++++++++++++++++++++-- src/ngx_json_log_output.c | 40 +++++++++++++++++++++++++++ src/ngx_json_log_output.h | 19 +++++++++++-- src/ngx_stream_json_log_module.c | 37 +++++++++++++++++++++++-- 6 files changed, 136 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ab79b74..7c44140 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,13 @@ The output format is configurable. It also allows to log complex and multi-level JSON documents. -It supports logging to text file or to a kafka topic. +It supports logging to text file, to syslog or to a kafka topic. It supports multiple output destinations with multiple formats for a location. ### Current version and limitations -Current version released is 0.0.6. +Current version released is 0.0.7. Stream logging is only available when using nginx (>= 1.11.2). @@ -118,6 +118,7 @@ Additionally, the variable $http_json_err_log_req will be set with a base64 enco '; json_log file:/tmp/log my_log;' + json_log syslog:server=unix:/dev/log my_log;' ``` This will produce the following JSON line to '/tmp/log' file . diff --git a/src/ngx_http_json_log_filter_module.c b/src/ngx_http_json_log_filter_module.c index dbb0883..5aa3434 100644 --- a/src/ngx_http_json_log_filter_module.c +++ b/src/ngx_http_json_log_filter_module.c @@ -437,6 +437,18 @@ ngx_http_json_log_header_bad_request(ngx_http_request_t *r) continue; } + /* Write to syslog */ + if (location->type == NGX_JSON_LOG_SINK_SYSLOG) { + if (!location->syslog) { + continue; + } + if (ngx_json_log_write_sink_syslog(r->pool->log, + r->pool, location->syslog, txt) == NGX_ERROR) { + ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, "Syslog write error!"); + } + continue; + } + #if (NGX_HAVE_LIBRDKAFKA) /* Write to kafka */ if (location->type == NGX_JSON_LOG_SINK_KAFKA) { diff --git a/src/ngx_http_json_log_module.c b/src/ngx_http_json_log_module.c index d712a0b..2806aef 100644 --- a/src/ngx_http_json_log_module.c +++ b/src/ngx_http_json_log_module.c @@ -287,7 +287,19 @@ static ngx_int_t ngx_http_json_log_log_handler(ngx_http_request_t *r) if (ngx_json_log_write_sink_file(r->pool->log, location->file->fd, txt) == NGX_ERROR) { - ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, "Write Error!"); + ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, "File write error!"); + } + continue; + } + + /* Write to syslog */ + if (location->type == NGX_JSON_LOG_SINK_SYSLOG) { + if (!location->syslog) { + continue; + } + if (ngx_json_log_write_sink_syslog(r->pool->log, + r->pool, location->syslog, txt) == NGX_ERROR) { + ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, "Syslog write error!"); } continue; } @@ -472,6 +484,7 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) #if (NGX_HAVE_LIBRDKAFKA) && ! NGX_JSON_LOG_HAS_KAFKA_PREFIX(value) #endif + && ! NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value) ) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "Invalid prefix [%v] for HTTP log JSON output location", value); @@ -491,11 +504,13 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) new_location->type = NGX_JSON_LOG_SINK_FILE; prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; #if (NGX_HAVE_LIBRDKAFKA) - } - else if (NGX_JSON_LOG_HAS_KAFKA_PREFIX(value)) { + } else if (NGX_JSON_LOG_HAS_KAFKA_PREFIX(value)) { new_location->type = NGX_JSON_LOG_SINK_KAFKA; prefix_len = NGX_JSON_LOG_KAFKA_OUT_LEN; #endif + } else if (NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value)) { + new_location->type = NGX_JSON_LOG_SINK_SYSLOG; + prefix_len = NGX_JSON_LOG_SYSLOG_OUT_LEN; } /* Saves location without prefix. */ @@ -510,6 +525,18 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) &new_location->location); } + /* If sink type is syslog */ + if (new_location->type == NGX_JSON_LOG_SINK_SYSLOG) { + new_location->syslog = ngx_pcalloc(cf->pool, sizeof(ngx_syslog_peer_t)); + if (new_location->syslog == NULL) { + return NGX_CONF_ERROR; + } + + if (ngx_syslog_process_conf(cf, new_location->syslog) != NGX_CONF_OK) { + return NGX_CONF_ERROR; + } + } + #if (NGX_HAVE_LIBRDKAFKA) /* If sink type is kafka, then set topic config for this location */ if (new_location->type == NGX_JSON_LOG_SINK_KAFKA) { diff --git a/src/ngx_json_log_output.c b/src/ngx_json_log_output.c index 7043ea8..309b3ef 100644 --- a/src/ngx_json_log_output.c +++ b/src/ngx_json_log_output.c @@ -19,3 +19,43 @@ ngx_json_log_write_sink_file(ngx_log_t *log, } return NGX_OK; } + +ngx_int_t +ngx_json_log_write_sink_syslog(ngx_log_t *log, + ngx_pool_t* pool, + ngx_syslog_peer_t *syslog, + const char *txt) { + + ssize_t n = 0; + size_t size = 0; + u_char *line, *p; + size_t len = 0; + + len += sizeof("<255>Jan 01 00:00:00 ") - 1 + + ngx_cycle->hostname.len + 1 + + syslog->tag.len + 2 + + strlen(txt); + + line = ngx_pnalloc(pool, len); + if (line == NULL) { + return NGX_ERROR; + } + + p = ngx_syslog_add_header(syslog, line); + p = ngx_snprintf(p, len, "%s", txt); + size = p - line; + n = ngx_syslog_send(syslog, line, size); + + if (n < 0) { + ngx_log_error(NGX_LOG_WARN, log, 0, + "send() to syslog failed"); + return NGX_ERROR; + + } else if ((size_t) n != size) { + ngx_log_error(NGX_LOG_WARN, log, 0, + "send() to syslog has written only %z of %uz", + n, size); + return NGX_ERROR; + } + return NGX_OK; +} diff --git a/src/ngx_json_log_output.h b/src/ngx_json_log_output.h index 285cdb6..2f6ec68 100644 --- a/src/ngx_json_log_output.h +++ b/src/ngx_json_log_output.h @@ -44,10 +44,18 @@ NGX_JSON_LOG_KAFKA_OUT_LEN) == 0 ) #endif +#define NGX_JSON_LOG_SYSLOG_OUT_LEN (sizeof("syslog:") - 1) +#define NGX_JSON_LOG_HAS_SYSLOG_PREFIX(str) \ + (ngx_strncmp(str->data, \ + "syslog:", \ + NGX_JSON_LOG_SYSLOG_OUT_LEN) == 0 ) + + typedef enum { NGX_JSON_LOG_SINK_FILE = 0, + NGX_JSON_LOG_SINK_SYSLOG = 1, #if (NGX_HAVE_LIBRDKAFKA) - NGX_JSON_LOG_SINK_KAFKA = 1 + NGX_JSON_LOG_SINK_KAFKA = 2 #endif } ngx_json_log_sink_e; @@ -55,7 +63,8 @@ struct ngx_json_log_output_location_s { ngx_str_t location; ngx_json_log_sink_e type; ngx_json_log_format_t format; - ngx_open_file_t *file; + ngx_open_file_t *file; + ngx_syslog_peer_t *syslog; #if (NGX_HAVE_LIBRDKAFKA) ngx_json_log_kafka_conf_t kafka; #endif @@ -67,5 +76,11 @@ ngx_int_t ngx_json_log_write_sink_file(ngx_log_t *log, ngx_fd_t fd, const char *txt); +ngx_int_t +ngx_json_log_write_sink_syslog(ngx_log_t *log, + ngx_pool_t* pool, + ngx_syslog_peer_t *syslog, + const char *txt); + #endif // __NGX_JSON_LOG_OUTPUT_H__ diff --git a/src/ngx_stream_json_log_module.c b/src/ngx_stream_json_log_module.c index 7910b86..bf8037b 100644 --- a/src/ngx_stream_json_log_module.c +++ b/src/ngx_stream_json_log_module.c @@ -237,7 +237,19 @@ ngx_stream_json_log_log_handler(ngx_stream_session_t *s) if (ngx_json_log_write_sink_file(s->connection->pool->log, location->file->fd, txt) == NGX_ERROR) { ngx_log_error(NGX_LOG_EMERG, - s->connection->pool->log, 0, "Write Error!"); + s->connection->pool->log, 0, "File write error!"); + } + continue; + } + + /* Write to syslog */ + if (location->type == NGX_JSON_LOG_SINK_SYSLOG) { + if (!location->syslog) { + continue; + } + if (ngx_json_log_write_sink_syslog(s->connection->pool->log, + s->connection->pool,location->syslog, txt) == NGX_ERROR) { + ngx_log_error(NGX_LOG_EMERG, s->connection->pool->log, 0, "Syslog write error!"); } continue; } @@ -393,8 +405,12 @@ ngx_stream_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) value = &args[1]; - if (! NGX_JSON_LOG_HAS_FILE_PREFIX(value) && - ! NGX_JSON_LOG_HAS_KAFKA_PREFIX(value)) { + if (! NGX_JSON_LOG_HAS_FILE_PREFIX(value) +#if (NGX_HAVE_LIBRDKAFKA) + && ! NGX_JSON_LOG_HAS_KAFKA_PREFIX(value) +#endif + && ! NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value) + ) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "Invalid prefix [%v] for HTTP log JSON output location", value); return NGX_CONF_ERROR; @@ -418,6 +434,9 @@ ngx_stream_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) new_location->type = NGX_JSON_LOG_SINK_KAFKA; prefix_len = NGX_JSON_LOG_KAFKA_OUT_LEN; #endif + } else if (NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value)) { + new_location->type = NGX_JSON_LOG_SINK_SYSLOG; + prefix_len = NGX_JSON_LOG_SYSLOG_OUT_LEN; } /* Saves location without prefix. */ @@ -432,6 +451,18 @@ ngx_stream_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) &new_location->location); } + /* If sink type is syslog */ + if (new_location->type == NGX_JSON_LOG_SINK_SYSLOG) { + new_location->syslog = ngx_pcalloc(cf->pool, sizeof(ngx_syslog_peer_t)); + if (new_location->syslog == NULL) { + return NGX_CONF_ERROR; + } + + if (ngx_syslog_process_conf(cf, new_location->syslog) != NGX_CONF_OK) { + return NGX_CONF_ERROR; + } + } + #if (NGX_HAVE_LIBRDKAFKA) /* If sink type is kafka, then set topic config for this location */ if (new_location->type == NGX_JSON_LOG_SINK_KAFKA) { From eed8c9e59db128a187fa664a9e3c08150b58742e Mon Sep 17 00:00:00 2001 From: fooinha Date: 2017年12月23日 12:10:41 +0000 Subject: [PATCH 2/3] ngx-json-log: refactor output location --- src/ngx_http_json_log_filter_module.c | 63 ++------------ src/ngx_http_json_log_module.c | 82 ++---------------- src/ngx_json_log_output.c | 118 ++++++++++++++++++++++++++ src/ngx_json_log_output.h | 31 +++---- src/ngx_stream_json_log_module.c | 102 ++-------------------- 5 files changed, 145 insertions(+), 251 deletions(-) diff --git a/src/ngx_http_json_log_filter_module.c b/src/ngx_http_json_log_filter_module.c index 5aa3434..94d42e0 100644 --- a/src/ngx_http_json_log_filter_module.c +++ b/src/ngx_http_json_log_filter_module.c @@ -767,15 +767,10 @@ static char * ngx_http_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { ngx_http_json_log_srv_conf_t *lc = conf; - ngx_http_json_log_main_conf_t *mcf; - ngx_json_log_output_location_t *new_location = NULL; + ngx_http_json_log_main_conf_t *mcf = NULL; ngx_json_log_format_t *format; ngx_str_t *args = cf->args->elts; - ngx_str_t *value = NULL; - ngx_str_t *format_name; - ngx_uint_t found = 0; - size_t prefix_len; - size_t i; + ngx_json_log_output_location_t *new_location; if (! args) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, @@ -791,66 +786,20 @@ ngx_http_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - /* Check if format exists by name */ - format_name = &args[2]; - format = mcf->formats->elts; - for (i = 0; i < mcf->formats->nelts; i++) { - if (ngx_strncmp(format_name->data, format[i].name.data, - format[i].name.len) == 0) { - found = 1; - break; - } - } - + format = ngx_json_log_check_format(mcf->formats, &args[2]); /* Do not accept unknown format names */ - if (!found) { + if (format == NULL) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "http_json_log: Invalid format name [%V]", - format_name); - return NGX_CONF_ERROR; - } - - value = &args[1]; - - if (! NGX_JSON_LOG_HAS_FILE_PREFIX(value) -#if (NGX_HAVE_LIBRDKAFKA) - && ! NGX_JSON_LOG_HAS_KAFKA_PREFIX(value) -#endif - ) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "Invalid prefix [%v] for HTTP log JSON output location", value); + &args[2]); return NGX_CONF_ERROR; } - new_location = ngx_array_push(lc->locations); + new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, &args[1]); if (new_location == NULL) { return NGX_CONF_ERROR; } - ngx_memzero(new_location, sizeof(*new_location)); - prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; - if (NGX_JSON_LOG_HAS_FILE_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_FILE; - prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; -#if (NGX_HAVE_LIBRDKAFKA) - } - else if (NGX_JSON_LOG_HAS_KAFKA_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_KAFKA; - prefix_len = NGX_JSON_LOG_KAFKA_OUT_LEN; -#endif - } - - /* Saves location without prefix. */ - new_location->location = args[1]; - new_location->location.len -= prefix_len; - new_location->location.data += prefix_len; - new_location->format = format[i]; - - /* If sink type is file, then try to open it and save */ - if (new_location->type == NGX_JSON_LOG_SINK_FILE) { - new_location->file = ngx_conf_open_file(cf->cycle, - &new_location->location); - } #if (NGX_HAVE_LIBRDKAFKA) /* If sink type is kafka, then set topic config for this location */ diff --git a/src/ngx_http_json_log_module.c b/src/ngx_http_json_log_module.c index 2806aef..70a0002 100644 --- a/src/ngx_http_json_log_module.c +++ b/src/ngx_http_json_log_module.c @@ -437,14 +437,9 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { ngx_http_json_log_loc_conf_t *lc = conf; ngx_http_json_log_main_conf_t *mcf = NULL; - ngx_json_log_output_location_t *new_location = NULL; ngx_json_log_format_t *format; ngx_str_t *args = cf->args->elts; - ngx_str_t *value = NULL; - ngx_str_t *format_name; - ngx_uint_t found = 0; - size_t prefix_len; - size_t i; + ngx_json_log_output_location_t *new_location; if (! args) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, @@ -459,83 +454,19 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - /* Check if format exists by name */ - format_name = &args[2]; - format = mcf->formats->elts; - for (i = 0; i < mcf->formats->nelts; i++) { - if (ngx_strncmp(format_name->data, format[i].name.data, - format[i].name.len) == 0) { - found = 1; - break; - } - } - + format = ngx_json_log_check_format(mcf->formats, &args[2]); /* Do not accept unknown format names */ - if (!found) { + if (format == NULL) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "http_json_log: Invalid format name [%V]", - format_name); - return NGX_CONF_ERROR; - } - - value = &args[1]; - - if (! NGX_JSON_LOG_HAS_FILE_PREFIX(value) -#if (NGX_HAVE_LIBRDKAFKA) - && ! NGX_JSON_LOG_HAS_KAFKA_PREFIX(value) -#endif - && ! NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value) - ) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "Invalid prefix [%v] for HTTP log JSON output location", value); + &args[2]); return NGX_CONF_ERROR; } - new_location = ngx_array_push(lc->locations); - if (!new_location) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "Failed to add [%v] for HTTP log JSON output location", value); + new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, &args[1]); + if (new_location == NULL) { return NGX_CONF_ERROR; } - ngx_memzero(new_location, sizeof(*new_location)); - - prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; - if (NGX_JSON_LOG_HAS_FILE_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_FILE; - prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; -#if (NGX_HAVE_LIBRDKAFKA) - } else if (NGX_JSON_LOG_HAS_KAFKA_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_KAFKA; - prefix_len = NGX_JSON_LOG_KAFKA_OUT_LEN; -#endif - } else if (NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_SYSLOG; - prefix_len = NGX_JSON_LOG_SYSLOG_OUT_LEN; - } - - /* Saves location without prefix. */ - new_location->location = args[1]; - new_location->location.len -= prefix_len; - new_location->location.data += prefix_len; - new_location->format = format[i]; - - /* If sink type is file, then try to open it and save */ - if (new_location->type == NGX_JSON_LOG_SINK_FILE) { - new_location->file = ngx_conf_open_file(cf->cycle, - &new_location->location); - } - - /* If sink type is syslog */ - if (new_location->type == NGX_JSON_LOG_SINK_SYSLOG) { - new_location->syslog = ngx_pcalloc(cf->pool, sizeof(ngx_syslog_peer_t)); - if (new_location->syslog == NULL) { - return NGX_CONF_ERROR; - } - - if (ngx_syslog_process_conf(cf, new_location->syslog) != NGX_CONF_OK) { - return NGX_CONF_ERROR; - } - } #if (NGX_HAVE_LIBRDKAFKA) /* If sink type is kafka, then set topic config for this location */ @@ -562,7 +493,6 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) /* Set variable for message id */ ngx_memzero(&ccv, sizeof(ngx_http_compile_complex_value_t)); - ccv.cf = cf; ccv.value = &msg_id_variable; ccv.complex_value = ngx_pcalloc(cf->pool, diff --git a/src/ngx_json_log_output.c b/src/ngx_json_log_output.c index 309b3ef..1ee54a8 100644 --- a/src/ngx_json_log_output.c +++ b/src/ngx_json_log_output.c @@ -1,5 +1,123 @@ #include "ngx_json_log_output.h" +#include + + +#define NGX_JSON_LOG_FILE_OUT_LEN (sizeof("file:") - 1) +#define NGX_JSON_LOG_HAS_FILE_PREFIX(str) \ + (ngx_strncmp(str->data, \ + "file:", \ + NGX_JSON_LOG_FILE_OUT_LEN) == 0 ) + +#if (NGX_HAVE_LIBRDKAFKA) +#define NGX_JSON_LOG_KAFKA_OUT_LEN (sizeof("kafka:") - 1) +#define NGX_JSON_LOG_HAS_KAFKA_PREFIX(str) \ + (ngx_strncmp(str->data, \ + "kafka:", \ + NGX_JSON_LOG_KAFKA_OUT_LEN) == 0 ) +#endif + +#define NGX_JSON_LOG_SYSLOG_OUT_LEN (sizeof("syslog:") - 1) +#define NGX_JSON_LOG_HAS_SYSLOG_PREFIX(str) \ + (ngx_strncmp(str->data, \ + "syslog:", \ + NGX_JSON_LOG_SYSLOG_OUT_LEN) == 0 ) + + + +/* Check if format exists by name */ +ngx_json_log_format_t * +ngx_json_log_check_format(ngx_array_t *formats, + ngx_str_t *name) +{ + ngx_json_log_format_t *format; + size_t i; + + /* Check if format exists by name */ + format = formats->elts; + for (i = 0; i < formats->nelts; i++) { + if (ngx_strncmp(name->data, format[i].name.data, + format[i].name.len) == 0) { + return &format[i]; + } + } + + return NULL; +} + +ngx_json_log_output_location_t * +ngx_json_log_output_location_conf(ngx_conf_t *cf, + ngx_json_log_format_t *format, + ngx_array_t *locations, + ngx_str_t *value) +{ + ngx_json_log_output_location_t *new_location = NULL; + size_t prefix_len; + + if (! NGX_JSON_LOG_HAS_FILE_PREFIX(value) +#if (NGX_HAVE_LIBRDKAFKA) + && ! NGX_JSON_LOG_HAS_KAFKA_PREFIX(value) +#endif + && ! NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value) + ) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "Invalid prefix [%v] JSON log output location", value); + return NULL; + } + + new_location = ngx_array_push(locations); + if (!new_location) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "Failed to add [%v] JSON log output location", value); + return NULL; + } + ngx_memzero(new_location, sizeof(*new_location)); + + prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; + if (NGX_JSON_LOG_HAS_FILE_PREFIX(value)) { + new_location->type = NGX_JSON_LOG_SINK_FILE; + prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; +#if (NGX_HAVE_LIBRDKAFKA) + } else if (NGX_JSON_LOG_HAS_KAFKA_PREFIX(value)) { + new_location->type = NGX_JSON_LOG_SINK_KAFKA; + prefix_len = NGX_JSON_LOG_KAFKA_OUT_LEN; +#endif + } else if (NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value)) { + new_location->type = NGX_JSON_LOG_SINK_SYSLOG; + prefix_len = NGX_JSON_LOG_SYSLOG_OUT_LEN; + } + + /* Saves location without prefix. */ + new_location->location = *value; + new_location->location.len -= prefix_len; + new_location->location.data += prefix_len; + new_location->format = *format; + + /* If sink type is file, then try to open it and save */ + if (new_location->type == NGX_JSON_LOG_SINK_FILE) { + new_location->file = ngx_conf_open_file(cf->cycle, + &new_location->location); + } + + /* If sink type is syslog */ + if (new_location->type == NGX_JSON_LOG_SINK_SYSLOG) { + new_location->syslog = ngx_pcalloc(cf->pool, sizeof(ngx_syslog_peer_t)); + if (new_location->syslog == NULL) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "Failed to allocated JSON log output location to syslog"); + return NULL; + } + + if (ngx_syslog_process_conf(cf, new_location->syslog) != NGX_CONF_OK) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "Failed to process syslog JSON log output location"); + return NULL; + } + } + + return new_location; +} + ngx_int_t ngx_json_log_write_sink_file(ngx_log_t *log, ngx_fd_t fd, const char *txt) diff --git a/src/ngx_json_log_output.h b/src/ngx_json_log_output.h index 2f6ec68..65e563b 100644 --- a/src/ngx_json_log_output.h +++ b/src/ngx_json_log_output.h @@ -30,27 +30,6 @@ #include "ngx_json_log_text.h" #include "ngx_json_log_kafka.h" -#define NGX_JSON_LOG_FILE_OUT_LEN (sizeof("file:") - 1) -#define NGX_JSON_LOG_HAS_FILE_PREFIX(str) \ - (ngx_strncmp(str->data, \ - "file:", \ - NGX_JSON_LOG_FILE_OUT_LEN) == 0 ) - -#if (NGX_HAVE_LIBRDKAFKA) -#define NGX_JSON_LOG_KAFKA_OUT_LEN (sizeof("kafka:") - 1) -#define NGX_JSON_LOG_HAS_KAFKA_PREFIX(str) \ - (ngx_strncmp(str->data, \ - "kafka:", \ - NGX_JSON_LOG_KAFKA_OUT_LEN) == 0 ) -#endif - -#define NGX_JSON_LOG_SYSLOG_OUT_LEN (sizeof("syslog:") - 1) -#define NGX_JSON_LOG_HAS_SYSLOG_PREFIX(str) \ - (ngx_strncmp(str->data, \ - "syslog:", \ - NGX_JSON_LOG_SYSLOG_OUT_LEN) == 0 ) - - typedef enum { NGX_JSON_LOG_SINK_FILE = 0, NGX_JSON_LOG_SINK_SYSLOG = 1, @@ -72,6 +51,16 @@ struct ngx_json_log_output_location_s { typedef struct ngx_json_log_output_location_s ngx_json_log_output_location_t; +ngx_json_log_format_t * +ngx_json_log_check_format(ngx_array_t *formats, + ngx_str_t *name); + +ngx_json_log_output_location_t * +ngx_json_log_output_location_conf(ngx_conf_t *cf, + ngx_json_log_format_t *format, + ngx_array_t *locations, + ngx_str_t *value); + ngx_int_t ngx_json_log_write_sink_file(ngx_log_t *log, ngx_fd_t fd, const char *txt); diff --git a/src/ngx_stream_json_log_module.c b/src/ngx_stream_json_log_module.c index bf8037b..1f39478 100644 --- a/src/ngx_stream_json_log_module.c +++ b/src/ngx_stream_json_log_module.c @@ -364,11 +364,6 @@ ngx_stream_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) ngx_json_log_output_location_t *new_location = NULL; ngx_json_log_format_t *format; ngx_str_t *args = cf->args->elts; - ngx_str_t *value = NULL; - ngx_str_t *format_name; - ngx_uint_t found = 0; - size_t prefix_len; - size_t i; if (! args) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, @@ -383,85 +378,20 @@ ngx_stream_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - - /* Check if format exists by name */ - format_name = &args[2]; - format = mcf->formats->elts; - for (i = 0; i < mcf->formats->nelts; i++) { - if (ngx_strncmp(format_name->data, format[i].name.data, - format[i].name.len) == 0) { - found = 1; - break; - } - } - + format = ngx_json_log_check_format(mcf->formats, &args[2]); /* Do not accept unknown format names */ - if (!found) { + if (format == NULL) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "http_json_log: Invalid format name [%V]", - format_name); - return NGX_CONF_ERROR; - } - - value = &args[1]; - - if (! NGX_JSON_LOG_HAS_FILE_PREFIX(value) -#if (NGX_HAVE_LIBRDKAFKA) - && ! NGX_JSON_LOG_HAS_KAFKA_PREFIX(value) -#endif - && ! NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value) - ) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "Invalid prefix [%v] for HTTP log JSON output location", value); + &args[2]); return NGX_CONF_ERROR; } - new_location = ngx_array_push(lc->locations); - if (!new_location) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "Failed to add [%v] for HTTP log JSON output location", value); + new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, &args[1]); + if (new_location == NULL) { return NGX_CONF_ERROR; } - ngx_memzero(new_location, sizeof(*new_location)); - - prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; - if (NGX_JSON_LOG_HAS_FILE_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_FILE; - prefix_len = NGX_JSON_LOG_FILE_OUT_LEN; -#if (NGX_HAVE_LIBRDKAFKA) - } - else if (NGX_JSON_LOG_HAS_KAFKA_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_KAFKA; - prefix_len = NGX_JSON_LOG_KAFKA_OUT_LEN; -#endif - } else if (NGX_JSON_LOG_HAS_SYSLOG_PREFIX(value)) { - new_location->type = NGX_JSON_LOG_SINK_SYSLOG; - prefix_len = NGX_JSON_LOG_SYSLOG_OUT_LEN; - } - - /* Saves location without prefix. */ - new_location->location = args[1]; - new_location->location.len -= prefix_len; - new_location->location.data += prefix_len; - new_location->format = format[i]; - - /* If sink type is file, then try to open it and save */ - if (new_location->type == NGX_JSON_LOG_SINK_FILE) { - new_location->file = ngx_conf_open_file(cf->cycle, - &new_location->location); - } - /* If sink type is syslog */ - if (new_location->type == NGX_JSON_LOG_SINK_SYSLOG) { - new_location->syslog = ngx_pcalloc(cf->pool, sizeof(ngx_syslog_peer_t)); - if (new_location->syslog == NULL) { - return NGX_CONF_ERROR; - } - - if (ngx_syslog_process_conf(cf, new_location->syslog) != NGX_CONF_OK) { - return NGX_CONF_ERROR; - } - } #if (NGX_HAVE_LIBRDKAFKA) /* If sink type is kafka, then set topic config for this location */ @@ -480,28 +410,6 @@ ngx_stream_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) /* Set global variable */ stream_json_log_has_kafka_locations = NGX_OK; - -//FIXME: #if nginx_version>= 1011000 -//FIXME: ngx_http_compile_complex_value_t ccv; -//FIXME: /*FIXME: Change this to an user's configured variable */ -//FIXME: ngx_str_t msg_id_variable = ngx_string("$request_id"); -//FIXME: -//FIXME: /* Set variable for message id */ -//FIXME: ngx_memzero(&ccv, sizeof(ngx_stream_compile_complex_value_t)); -//FIXME: -//FIXME: -//FIXME: ccv.cf = cf; -//FIXME: ccv.value = &msg_id_variable; -//FIXME: ccv.complex_value = ngx_pcalloc(cf->pool, -//FIXME: sizeof(ngx_stream_complex_value_t)); -//FIXME: if (ccv.complex_value == NULL) { -//FIXME: return NGX_CONF_ERROR; -//FIXME: } -//FIXME: if (ngx_stream_compile_complex_value(&ccv) != NGX_OK) { -//FIXME: return NGX_CONF_ERROR; -//FIXME: } -//FIXME: new_location->kafka.stream_msg_id_var = ccv.complex_value; -//FIXME: #endif } #endif From 0ad2cb9c731364577b7018431f8df6b1307f9857 Mon Sep 17 00:00:00 2001 From: fooinha Date: 2017年12月23日 13:20:26 +0000 Subject: [PATCH 3/3] ngx-json-log: styling fixes --- docker/debian-nginx-http-log-json/vimrc | 4 ++-- src/ngx_http_json_log_filter_module.c | 29 ++++++++++++++----------- src/ngx_http_json_log_module.c | 9 +++++--- src/ngx_stream_json_log_module.c | 18 ++++++++------- 4 files changed, 34 insertions(+), 26 deletions(-) diff --git a/docker/debian-nginx-http-log-json/vimrc b/docker/debian-nginx-http-log-json/vimrc index cedc851..814843a 100644 --- a/docker/debian-nginx-http-log-json/vimrc +++ b/docker/debian-nginx-http-log-json/vimrc @@ -77,8 +77,8 @@ autocmd BufWinLeave * call clearmatches() " Source a global configuration file if available if filereadable("/etc/vim/vimrc.local") - source /etc/vim/vimrc.local - endif + source /etc/vim/vimrc.local +endif " CTags tag on cursor find definition map ? :tag =expand("") diff --git a/src/ngx_http_json_log_filter_module.c b/src/ngx_http_json_log_filter_module.c index 94d42e0..1d300b8 100644 --- a/src/ngx_http_json_log_filter_module.c +++ b/src/ngx_http_json_log_filter_module.c @@ -178,7 +178,8 @@ static ngx_command_t ngx_http_json_log_filter_commands[] = { NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, ngx_conf_set_num_slot, NGX_HTTP_MAIN_CONF_OFFSET, - offsetof(ngx_http_json_log_filter_main_conf_t, kafka.buffer_max_messages), + offsetof(ngx_http_json_log_filter_main_conf_t, + kafka.buffer_max_messages), NULL }, { @@ -195,14 +196,14 @@ static ngx_command_t ngx_http_json_log_filter_commands[] = { /* config preparation */ static ngx_http_module_t ngx_http_json_log_filter_module_ctx = { - NULL, /* preconfiguration */ - ngx_http_json_log_filter_init, /* postconfiguration */ - ngx_http_json_log_filter_create_main_conf, /* create main configuration */ - NULL, /* init main configuration */ - ngx_http_json_log_filter_create_srv_conf, /* create server configuration */ - NULL, /* merge server configuration */ - ngx_http_json_log_filter_create_loc_conf, /* create location configuration */ - NULL /* merge location configuration */ + NULL, /* preconfiguration */ + ngx_http_json_log_filter_init, /* postconfiguration */ + ngx_http_json_log_filter_create_main_conf,/* create main configuration */ + NULL, /* init main configuration */ + ngx_http_json_log_filter_create_srv_conf, /* create server configuration */ + NULL, /* merge server configuration */ + ngx_http_json_log_filter_create_loc_conf, /* create location configuration*/ + NULL /* merge location configuration */ }; ngx_module_t ngx_http_json_log_filter_module = { @@ -444,7 +445,8 @@ ngx_http_json_log_header_bad_request(ngx_http_request_t *r) } if (ngx_json_log_write_sink_syslog(r->pool->log, r->pool, location->syslog, txt) == NGX_ERROR) { - ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, "Syslog write error!"); + ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, + "Syslog write error!"); } continue; } @@ -673,7 +675,8 @@ ngx_http_json_log_body_filter(ngx_http_request_t *r, ngx_chain_t *in) if (!vv_hex) { return ngx_http_next_header_filter(r); } - ngx_http_json_log_set_variable_req_body_hexdump(r, vv_hex, (uintptr_t) &payload); + ngx_http_json_log_set_variable_req_body_hexdump(r, + vv_hex, (uintptr_t) &payload); return ngx_http_next_request_body_filter(r, in); } @@ -795,7 +798,8 @@ ngx_http_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, &args[1]); + new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, + &args[1]); if (new_location == NULL) { return NGX_CONF_ERROR; } @@ -864,7 +868,6 @@ ngx_http_json_log_filter_init_worker(ngx_cycle_t *cycle) return NGX_OK; } - rc = ngx_json_log_configure_kafka(cycle->pool, &conf->kafka); if (rc != NGX_OK) { return NGX_OK; //FIXME: What to do? diff --git a/src/ngx_http_json_log_module.c b/src/ngx_http_json_log_module.c index 70a0002..48cb59b 100644 --- a/src/ngx_http_json_log_module.c +++ b/src/ngx_http_json_log_module.c @@ -287,7 +287,8 @@ static ngx_int_t ngx_http_json_log_log_handler(ngx_http_request_t *r) if (ngx_json_log_write_sink_file(r->pool->log, location->file->fd, txt) == NGX_ERROR) { - ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, "File write error!"); + ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, + "File write error!"); } continue; } @@ -299,7 +300,8 @@ static ngx_int_t ngx_http_json_log_log_handler(ngx_http_request_t *r) } if (ngx_json_log_write_sink_syslog(r->pool->log, r->pool, location->syslog, txt) == NGX_ERROR) { - ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, "Syslog write error!"); + ngx_log_error(NGX_LOG_EMERG, r->pool->log, 0, + "Syslog write error!"); } continue; } @@ -463,7 +465,8 @@ ngx_http_json_log_loc_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, &args[1]); + new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, + &args[1]); if (new_location == NULL) { return NGX_CONF_ERROR; } diff --git a/src/ngx_stream_json_log_module.c b/src/ngx_stream_json_log_module.c index 1f39478..6149982 100644 --- a/src/ngx_stream_json_log_module.c +++ b/src/ngx_stream_json_log_module.c @@ -206,7 +206,8 @@ ngx_stream_json_log_log_handler(ngx_stream_session_t *s) /* Check filter result */ if (location->format.stream_filter != NULL) { if (ngx_stream_complex_value(s, - location->format.stream_filter, &filter_val) != NGX_OK) { + location->format.stream_filter, + &filter_val) != NGX_OK) { /* WARN ? */ continue; } @@ -248,8 +249,10 @@ ngx_stream_json_log_log_handler(ngx_stream_session_t *s) continue; } if (ngx_json_log_write_sink_syslog(s->connection->pool->log, - s->connection->pool,location->syslog, txt) == NGX_ERROR) { - ngx_log_error(NGX_LOG_EMERG, s->connection->pool->log, 0, "Syslog write error!"); + s->connection->pool, + location->syslog, txt) == NGX_ERROR) { + ngx_log_error(NGX_LOG_EMERG, s->connection->pool->log, 0, + "Syslog write error!"); } continue; } @@ -343,8 +346,6 @@ ngx_stream_json_log_post_conf(ngx_conf_t *cf) cmcf = ngx_stream_conf_get_module_main_conf(cf, ngx_stream_core_module); - //TODO-OPTIMIZATION: to verify if the module should be registered on log phase - h = ngx_array_push(&cmcf->phases[NGX_STREAM_LOG_PHASE].handlers); if (h == NULL) { return NGX_ERROR; @@ -387,12 +388,12 @@ ngx_stream_json_log_srv_output(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) return NGX_CONF_ERROR; } - new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, &args[1]); + new_location = ngx_json_log_output_location_conf(cf, format, lc->locations, + &args[1]); if (new_location == NULL) { return NGX_CONF_ERROR; } - #if (NGX_HAVE_LIBRDKAFKA) /* If sink type is kafka, then set topic config for this location */ if (new_location->type == NGX_JSON_LOG_SINK_KAFKA) { @@ -423,7 +424,8 @@ ngx_stream_json_log_init_worker(ngx_cycle_t *cycle) { #if (NGX_HAVE_LIBRDKAFKA) ngx_stream_json_log_main_conf_t *conf = - ngx_stream_cycle_get_module_main_conf(cycle, ngx_stream_json_log_module); + ngx_stream_cycle_get_module_main_conf(cycle, + ngx_stream_json_log_module); /* from this point we just are init kafka stuff */ if (stream_json_log_has_kafka_locations == NGX_CONF_UNSET ) {

AltStyle によって変換されたページ (->オリジナル) /