nginx中利用resolver实现动态upstream
nginx中如何利用resolver实现动态upstream呢？
首先了解下resolver，在nginx中，nginx有一套自己的域名解析过程，在nginx配置中，通过resolver指令来设置DNS服务器地址，来启动nginx的域名解析
首先通过源码看一下nginx是如何做的，本文基于nginx1.14.1版本分析
首先，resolver的初始化，在源码http中ngx_http_core_module.h中ngx_http_loc_conf_s的申明中可以看到对resolver的申明，在文件364行
resolver中保存了与域名解析相关的一些数据，它保存了DNS的本地缓存，通过红黑书的方式来组织数据，快速查找
在nginx初始化的时候，通过ngx_resolver_create来初始化，如果在配置文件中设置了resolver，则在ngx_http_core_resolver中有调用
在ngx_resolver_create的第二个参数，就是我们设置的域名解析服务器的IP地址
继续看一下ngx_resolver_create做了什么工作
ngx_resolver_create(ngx_conf_t *cf, ngx_str_t *names, ngx_uint_t n)
{
ngx_str_t s;
ngx_url_t u;
ngx_uint_t i, j;
ngx_resolver_t *r;
ngx_pool_cleanup_t *cln;
ngx_resolver_connection_t *rec;
cln = ngx_pool_cleanup_add(cf->pool, 0);
if (cln == NULL) {
return NULL;
}
cln->handler = ngx_resolver_cleanup;
r = ngx_calloc(sizeof(ngx_resolver_t), cf->log);
if (r == NULL) {
return NULL;
}
cln->data = r;
r->event = ngx_calloc(sizeof(ngx_event_t), cf->log);
if (r->event == NULL) {
return NULL;
}
ngx_rbtree_init(&r->name_rbtree, &r->name_sentinel,
ngx_resolver_rbtree_insert_value);
ngx_rbtree_init(&r->srv_rbtree, &r->srv_sentinel,
ngx_resolver_rbtree_insert_value);
ngx_rbtree_init(&r->addr_rbtree, &r->addr_sentinel,
ngx_rbtree_insert_value);
ngx_queue_init(&r->name_resend_queue);
ngx_queue_init(&r->srv_resend_queue);
ngx_queue_init(&r->addr_resend_queue);
ngx_queue_init(&r->name_expire_queue);
ngx_queue_init(&r->srv_expire_queue);
ngx_queue_init(&r->addr_expire_queue);
#if (NGX_HAVE_INET6)
r->ipv6 = 1;
ngx_rbtree_init(&r->addr6_rbtree, &r->addr6_sentinel,
ngx_resolver_rbtree_insert_addr6_value);
ngx_queue_init(&r->addr6_resend_queue);
ngx_queue_init(&r->addr6_expire_queue);
#endif
r->event->handler = ngx_resolver_resend_handler;
r->event->data = r;
r->event->log = &cf->cycle->new_log;
r->event->cancelable = 1;
r->ident = -1;
r->resend_timeout = 5;
r->tcp_timeout = 5;
r->expire = 30;
r->valid = 0;
r->log = &cf->cycle->new_log;
r->log_level = NGX_LOG_ERR;
if (n) {
if (ngx_array_init(&r->connections, cf->pool, n,
sizeof(ngx_resolver_connection_t))
!= NGX_OK)
{
return NULL;
}
}
for (i = 0; i < n; i++) {
if (ngx_strncmp(names[i].data, "valid=", 6) == 0) {
s.len = names[i].len - 6;
s.data = names[i].data + 6;
r->valid = ngx_parse_time(&s, 1);
if (r->valid == (time_t) NGX_ERROR) {
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"invalid parameter: %V", &names[i]);
return NULL;
}
continue;
}
#if (NGX_HAVE_INET6)
if (ngx_strncmp(names[i].data, "ipv6=", 5) == 0) {
if (ngx_strcmp(&names[i].data[5], "on") == 0) {
r->ipv6 = 1;
} else if (ngx_strcmp(&names[i].data[5], "off") == 0) {
r->ipv6 = 0;
} else {
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"invalid parameter: %V", &names[i]);
return NULL;
}
continue;
}
#endif
ngx_memzero(&u, sizeof(ngx_url_t));
u.url = names[i];
u.default_port = 53;
if (ngx_parse_url(cf->pool, &u) != NGX_OK) {
if (u.err) {
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"%s in resolver \"%V\"",
u.err, &u.url);
}
return NULL;
}
rec = ngx_array_push_n(&r->connections, u.naddrs);
if (rec == NULL) {
return NULL;
}
ngx_memzero(rec, u.naddrs * sizeof(ngx_resolver_connection_t));
for (j = 0; j < u.naddrs; j++) {
rec[j].sockaddr = u.addrs[j].sockaddr;
rec[j].socklen = u.addrs[j].socklen;
rec[j].server = u.addrs[j].name;
rec[j].resolver = r;
}
}
return r;
}
在resolver初始化完成之后，就可以调用了。在nginx中，upstream和proxy_pass中使用到了此方法的域名解析，所以下面结合这两个模块来看一下。
在proxy中，一般在配置文件中会配置proxy_pass变量，通过nginx的DNS对变量进行解析，上面的代码，是从ngx_http_proxy_handler中的对于proxy_pass变量部分的判断，可以看到，当没有变量的时候，是不进行域名解析的，只有当proxy_pass有变量的时候， 才会在ngx_http_proxy_eval中添加变量，进行域名解析，下面看下ngx_http_proxy_eval
ngx_http_proxy_eval(ngx_http_request_t *r, ngx_http_proxy_ctx_t *ctx,
ngx_http_proxy_loc_conf_t *plcf)
{
u_char *p;
size_t add;
u_short port;
ngx_str_t proxy;
ngx_url_t url;
ngx_http_upstream_t *u;
if (ngx_http_script_run(r, &proxy, plcf->proxy_lengths->elts, 0,
plcf->proxy_values->elts)
== NULL)
{
return NGX_ERROR;
}
//判断http or https，添加端口
if (proxy.len > 7
&& ngx_strncasecmp(proxy.data, (u_char *) "http://", 7) == 0)
{
add = 7;
port = 80;
#if (NGX_HTTP_SSL)
} else if (proxy.len > 8
&& ngx_strncasecmp(proxy.data, (u_char *) "https://", 8) == 0)
{
add = 8;
port = 443;
r->upstream->ssl = 1;
#endif
} else {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"invalid URL prefix in \"%V\"", &proxy);
return NGX_ERROR;
}
u = r->upstream;
u->schema.len = add;
u->schema.data = proxy.data;
ngx_memzero(&url, sizeof(ngx_url_t));
//proxy要转向的url
url.url.len = proxy.len - add;
url.url.data = proxy.data + add;
url.default_port = port;
url.uri_part = 1;
//不用域名解析
url.no_resolve = 1;
//上面配置不用域名解析，所以在ngx_parse_url中不会对域名进行解析
if (ngx_parse_url(r->pool, &url) != NGX_OK) {
if (url.err) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"%s in upstream \"%V\"", url.err, &url.url);
}
return NGX_ERROR;
}
if (url.uri.len) {
if (url.uri.data[0] == '?') {
p = ngx_pnalloc(r->pool, url.uri.len + 1);
if (p == NULL) {
return NGX_ERROR;
}
*p++ = '/';
ngx_memcpy(p, url.uri.data, url.uri.len);
url.uri.len++;
url.uri.data = p - 1;
}
}
ctx->vars.key_start = u->schema;
ngx_http_proxy_set_vars(&url, &ctx->vars);
//保存需要解析域名相关信息
u->resolved = ngx_pcalloc(r->pool, sizeof(ngx_http_upstream_resolved_t));
if (u->resolved == NULL) {
return NGX_ERROR;
}
if (url.addrs) {
//如果域名已经是ip地址的格式，直接保存，这样在upstream里面就不会再进行解析
//在upstream模块中会对u->resolved->sockaddr进行判断
u->resolved->sockaddr = url.addrs[0].sockaddr;
u->resolved->socklen = url.addrs[0].socklen;
u->resolved->name = url.addrs[0].name;
u->resolved->naddrs = 1;
}
u->resolved->host = url.host;
u->resolved->port = (in_port_t) (url.no_port ? port : url.port);
u->resolved->no_port = url.no_port;
return NGX_OK;
}
接下来在upstream中的ngx_http_upstream_init_request初始化请求时，当u->resolved不为空时，进行域名解析
然后开始查找域名，ngx_resolve_start初始化域名解析器，如果返回NGX_NO)RESOLVER无法进行域名解析
设置需要解析的域名，以及解析超时时间，handler解析完成后回调函数，然后ngx_resolve_name开始解析，超时没有解析完成，直接return
然后看一下ngx_resolve_star过程
ngx_resolver_ctx_t *
ngx_resolve_start(ngx_resolver_t *r, ngx_resolver_ctx_t *temp)
{
in_addr_t addr;
ngx_resolver_ctx_t *ctx;
if (temp) {
addr = ngx_inet_addr(temp->name.data, temp->name.len);
//如果要解析的地址已经是ip地址，则设置temp->quick为1，在ngx_resolve_name调用时就不会再进行解析
if (addr != INADDR_NONE) {
temp->resolver = r;
temp->state = NGX_OK;
temp->naddrs = 1;
temp->addrs = &temp->addr;
temp->addr.sockaddr = (struct sockaddr *) &temp->sin;
temp->addr.socklen = sizeof(struct sockaddr_in);
ngx_memzero(&temp->sin, sizeof(struct sockaddr_in));
temp->sin.sin_family = AF_INET;
temp->sin.sin_addr.s_addr = addr;
//不需要进行域名解析
temp->quick = 1;
return temp;
}
}
//如果r->connections.nelts为0，则表示配置文件中没有配置dns服务器地址
if (r->connections.nelts == 0) {
return NGX_NO_RESOLVER;
}
ctx = ngx_resolver_calloc(r, sizeof(ngx_resolver_ctx_t));
if (ctx) {
ctx->resolver = r;
}
return ctx;
}
接着看下ngx_resolve_name解析的过程
ngx_int_t
ngx_resolve_name(ngx_resolver_ctx_t *ctx)
{
size_t slen;
ngx_int_t rc;
ngx_str_t name;
ngx_resolver_t *r;
r = ctx->resolver;
if (ctx->name.len > 0 && ctx->name.data[ctx->name.len - 1] == '.') {
ctx->name.len--;
}
ngx_log_debug1(NGX_LOG_DEBUG_CORE, r->log, 0,
"resolve: \"%V\"", &ctx->name);
//如果已经是IPi地址了，quick被设置为1，这里可以看到直接返回
if (ctx->quick) {
ctx->handler(ctx);
return NGX_OK;
}
//开始域名查找
if (ctx->service.len) {
slen = ctx->service.len;
if (ngx_strlchr(ctx->service.data,
ctx->service.data + ctx->service.len, '.')
== NULL)
{
slen += sizeof("_._tcp") - 1;
}
name.len = slen + 1 + ctx->name.len;
name.data = ngx_resolver_alloc(r, name.len);
if (name.data == NULL) {
goto failed;
}
if (slen == ctx->service.len) {
ngx_sprintf(name.data, "%V.%V", &ctx->service, &ctx->name);
} else {
ngx_sprintf(name.data, "_%V._tcp.%V", &ctx->service, &ctx->name);
}
/* lock name mutex */
rc = ngx_resolve_name_locked(r, ctx, &name);
ngx_resolver_free(r, name.data);
} else {
/* lock name mutex */
rc = ngx_resolve_name_locked(r, ctx, &ctx->name);
}
if (rc == NGX_OK) {
return NGX_OK;
}
/* unlock name mutex */
if (rc == NGX_AGAIN) {
return NGX_OK;
}
/* NGX_ERROR */
if (ctx->event) {
ngx_resolver_free(r, ctx->event);
}
failed:
ngx_resolver_free(r, ctx);
return NGX_ERROR;
}
在上面我们可以看到，调用ngx_resolve_name_locked来查找域名，接着看(这里代码很长，所以只截取部分重要的，感兴趣的可以去看一下源码)
可以看到，先在本地的DNS缓存中查找域名，如果本地缓存中能找到，则判断缓存时效，若没有超时，则更新DNS有效期，并移动到队列最前面
并设置解析状态和IP地址，执行回调函数
如果解析类型是CNAME，则回调查询IP地址，如果没有找到，则通过ngx_resolver_create_name_query创建新的DNS查询请求，并通过ngx_resolver_send_query发送请求，先看下创建请求
将DNS请求内容放入rn->query中，然后接着看ngx_resolver_send_query
static ngx_int_t
ngx_resolver_send_query(ngx_resolver_t *r, ngx_resolver_node_t *rn)
{
ngx_int_t rc;
ngx_resolver_connection_t *rec;
rec = r->connections.elts;
rec = &rec[rn->last_connection];
if (rec->log.handler == NULL) {
rec->log = *r->log;
rec->log.handler = ngx_resolver_log_error;
rec->log.data = rec;
rec->log.action = "resolving";
}
//判断采用TCP或UDP发送
if (rn->naddrs == (u_short) -1) {
rc = rn->tcp ? ngx_resolver_send_tcp_query(r, rec, rn->query, rn->qlen)
: ngx_resolver_send_udp_query(r, rec, rn->query, rn->qlen);
if (rc != NGX_OK) {
return rc;
}
}
#if (NGX_HAVE_INET6)
if (rn->query6 && rn->naddrs6 == (u_short) -1) {
rc = rn->tcp6
? ngx_resolver_send_tcp_query(r, rec, rn->query6, rn->qlen)
: ngx_resolver_send_udp_query(r, rec, rn->query6, rn->qlen);
if (rc != NGX_OK) {
return rc;
}
}
#endif
return NGX_OK;
}
采用UDP发送数据，并设置ngx_resolver_udp_read回调
采用TCP的方式和UDP的方式略有不同，当DNS服务器响应时，会调用ngx_resolver_udp_read或ngx_resolver_tcp_read函数，接收数据，并调用ngx_resolver_process_responese来处理响应
在ngx_resolver_process_response中根据响应的类别分别调用
ngx_resolver_process_a或ngx_resolver_process_srv或ngx_resolver_process_ptr，并将DNS缓存，且更新有效期，最后调用回调函数
在回调函数ngx_http_upstream_resolve_handler中
static void
ngx_http_upstream_resolve_handler(ngx_resolver_ctx_t *ctx)
{
ngx_uint_t run_posted;
ngx_connection_t *c;
ngx_http_request_t *r;
ngx_http_upstream_t *u;
ngx_http_upstream_resolved_t *ur;
run_posted = ctx->async;
r = ctx->data;
c = r->connection;
u = r->upstream;
ur = u->resolved;
ngx_http_set_log_request(c->log, r);
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,
"http upstream resolve: \"%V?%V\"", &r->uri, &r->args);
if (ctx->state) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"%V could not be resolved (%i: %s)",
&ctx->name, ctx->state,
ngx_resolver_strerror(ctx->state));
ngx_http_upstream_finalize_request(r, u, NGX_HTTP_BAD_GATEWAY);
goto failed;
}
ur->naddrs = ctx->naddrs;
ur->addrs = ctx->addrs;
#if (NGX_DEBUG)
{
u_char text[NGX_SOCKADDR_STRLEN];
ngx_str_t addr;
ngx_uint_t i;
addr.data = text;
for (i = 0; i < ctx->naddrs; i++) {
addr.len = ngx_sock_ntop(ur->addrs[i].sockaddr, ur->addrs[i].socklen,
text, NGX_SOCKADDR_STRLEN, 0);
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"name was resolved to %V", &addr);
}
}
#endif
if (ngx_http_upstream_create_round_robin_peer(r, ur) != NGX_OK) {
ngx_http_upstream_finalize_request(r, u,
NGX_HTTP_INTERNAL_SERVER_ERROR);
goto failed;
}
//结束DNS解析
ngx_resolve_name_done(ctx);
ur->ctx = NULL;
u->peer.start_time = ngx_current_msec;
if (u->conf->next_upstream_tries
&& u->peer.tries > u->conf->next_upstream_tries)
{
u->peer.tries = u->conf->next_upstream_tries;
}
//连接upstream
ngx_http_upstream_connect(r, u);
failed:
if (run_posted) {
ngx_http_run_posted_requests(c);
}
}
所以在nginx配置resolver后，会通过DNS内部的域名解析方法来进行域名解析。resolver的语法如下：
Syntax: resolver address ... [valid=time] [ipv6=on|off];Default: —Context: http, server, location
在resolver后面可以配置多个DNS地址，nginx会采用轮询的方式去访问，并对解析结果缓存，这里的valid就是指定缓存的时间。
另外有一个参数是配合resolver使用的，就是resolver_timeout，语法如下：
Syntax: resolver_timeout time;
Default: resolver_timeout 30s;
Context: http, server, location
该参数是用于指定DNS解析的超时时间。
理解了resolver的原理之后，利用resolver实现upstream就显而易见了，通过resolver指定nginx的DNS解析，在upstream中设置域名，反向代理后端，那么我们只需要修改域名的解析，就可以实现动态upstream，而无需重启nginx修改upstream配置。