NCBI C++ ToolKit
ncbi_dispd.c
Go to the documentation of this file.

Go to the SVN repository for this file.

1 /* $Id: ncbi_dispd.c 76512 2017-02-14 18:39:16Z lavr $
2  * ===========================================================================
3  *
4  * PUBLIC DOMAIN NOTICE
5  * National Center for Biotechnology Information
6  *
7  * This software/database is a "United States Government Work" under the
8  * terms of the United States Copyright Act. It was written as part of
9  * the author's official duties as a United States Government employee and
10  * thus cannot be copyrighted. This software/database is freely available
11  * to the public for use. The National Library of Medicine and the U.S.
12  * Government have not placed any restriction on its use or reproduction.
13  *
14  * Although all reasonable efforts have been taken to ensure the accuracy
15  * and reliability of the software and data, the NLM and the U.S.
16  * Government do not and cannot warrant the performance or results that
17  * may be obtained by using this software or data. The NLM and the U.S.
18  * Government disclaim all warranties, express or implied, including
19  * warranties of performance, merchantability or fitness for any particular
20  * purpose.
21  *
22  * Please cite the author in any work or product based on this material.
23  *
24  * ===========================================================================
25  *
26  * Author: Anton Lavrentiev
27  *
28  * File Description:
29  * Low-level API to resolve an NCBI service name to server meta-addresses
30  * with the use of the NCBI network dispatcher (DISPD).
31  *
32  */
33 
34 #include "ncbi_ansi_ext.h"
35 #include "ncbi_comm.h"
36 #include "ncbi_dispd.h"
37 #include "ncbi_lb.h"
38 #include "ncbi_priv.h"
40 #include <ctype.h>
41 #include <stdlib.h>
42 
43 #ifdef fabs
44 # undef fabs
45 #endif /*fabs*/
46 #define fabs(v) ((v) < 0.0 ? -(v) : (v))
47 
48 #define NCBI_USE_ERRCODE_X Connect_LBSM
49 
50 /* Lower bound of up-to-date/out-of-date ratio */
51 #define DISPD_STALE_RATIO_OK 0.8
52 /* Default rate increase 20% if svc runs locally */
53 #define DISPD_LOCAL_BONUS 1.2
54 
55 
56 #ifdef __cplusplus
57 extern "C" {
58 #endif /*__cplusplus*/
60 static int/*bool*/ s_Update (SERV_ITER, const char*, int);
61 static void s_Reset (SERV_ITER);
62 static void s_Close (SERV_ITER);
63 
64 static const SSERV_VTable s_op = {
65  s_GetNextInfo, 0/*Feedback*/, s_Update, s_Reset, s_Close, "DISPD"
66 };
67 #ifdef __cplusplus
68 } /* extern "C" */
69 #endif /*__cplusplus*/
70 
71 
72 struct SDISPD_Data {
73  short/*bool*/ eof; /* no more resolves */
74  short/*bool*/ fail; /* no more connects */
77  size_t n_cand;
78  size_t a_cand;
79  size_t n_skip;
80 };
81 
82 
83 static int/*bool*/ s_AddServerInfo(struct SDISPD_Data* data, SSERV_Info* info)
84 {
85  size_t i;
86  const char* name = SERV_NameOfInfo(info);
87  /* First check that the new server info updates an existing one */
88  for (i = 0; i < data->n_cand; i++) {
89  if (strcasecmp(name, SERV_NameOfInfo(data->cand[i].info)) == 0
90  && SERV_EqualInfo(info, data->cand[i].info)) {
91  /* Replace older version */
92  free((void*) data->cand[i].info);
93  data->cand[i].info = info;
94  return 1;
95  }
96  }
97  /* Next, add new service to the list */
98  if (data->n_cand == data->a_cand) {
99  size_t n = data->a_cand + 10;
100  SLB_Candidate* temp = (SLB_Candidate*)
101  (data->cand
102  ? realloc(data->cand, n * sizeof(*temp))
103  : malloc ( n * sizeof(*temp)));
104  if (!temp)
105  return 0;
106  data->cand = temp;
107  data->a_cand = n;
108  }
109  data->cand[data->n_cand++].info = info;
110  return 1;
111 }
112 
113 
114 #ifdef __cplusplus
115 extern "C" {
116  static EHTTP_HeaderParse s_ParseHeader(const char*, void*, int);
117 }
118 #endif /*__cplusplus*/
119 
120 static EHTTP_HeaderParse s_ParseHeader(const char* header,
121  void* iter,
122  int server_error)
123 {
124  struct SDISPD_Data* data = (struct SDISPD_Data*)((SERV_ITER) iter)->data;
125  int code = 0/*success code if any*/;
126  if (server_error) {
127  if (server_error == 400 || server_error == 403 || server_error == 404)
128  data->fail = 1/*true*/;
129  } else if (sscanf(header, "%*s %d", &code) < 1) {
130  data->eof = 1/*true*/;
131  return eHTTP_HeaderError;
132  }
133  /* check for empty document */
134  if (!SERV_Update((SERV_ITER) iter, header, server_error) || code == 204)
135  data->eof = 1/*true*/;
136  return eHTTP_HeaderSuccess;
137 }
138 
139 
140 #ifdef __cplusplus
141 extern "C" {
142  static int s_Adjust(SConnNetInfo*, void*, unsigned int);
143 }
144 #endif /*__cplusplus*/
145 
146 /*ARGSUSED*/
147 static int/*bool*/ s_Adjust(SConnNetInfo* net_info,
148  void* iter,
149  unsigned int unused)
150 {
151  struct SDISPD_Data* data = (struct SDISPD_Data*)((SERV_ITER) iter)->data;
152  return data->fail ? 0/*no more tries*/ : 1/*may try again*/;
153 }
154 
155 
156 static void s_Resolve(SERV_ITER iter)
157 {
158  struct SDISPD_Data* data = (struct SDISPD_Data*) iter->data;
159  SConnNetInfo* net_info = data->net_info;
160  EIO_Status status = eIO_Success;
161  CONNECTOR c = 0;
162  CONN conn;
163  char* s;
164 
165  assert(!(data->eof | data->fail));
166  assert(!!net_info->stateless == !!iter->stateless);
167  /* Obtain additional header information */
168  if ((!(s = SERV_Print(iter, 0, 0))
170  &&
172  iter->ok_down && iter->ok_suppressed
173  ? "Dispatch-Mode: PROMISCUOUS\r\n"
174  : iter->ok_down
175  ? "Dispatch-Mode: OK_DOWN\r\n"
176  : iter->ok_suppressed
177  ? "Dispatch-Mode: OK_SUPPRESSED\r\n"
178  : "Dispatch-Mode: INFORMATION_ONLY\r\n")
179  &&
181  ? "Client-Mode: REVERSE_DNS\r\n"
182  : !net_info->stateless
183  ? "Client-Mode: STATEFUL_CAPABLE\r\n"
184  : "Client-Mode: STATELESS_ONLY\r\n")) {
186  iter/*data*/, s_Adjust, 0/*cleanup*/);
187  }
188  if (s) {
190  free(s);
191  }
192  if (c && (status = CONN_Create(c, &conn)) == eIO_Success) {
193  /* Send all the HTTP data... */
194  CONN_Flush(conn);
195  /* ...then trigger the header callback */
196  CONN_Close(conn);
197  } else {
199  ("%s%s%sUnable to create auxiliary HTTP %s: %s",
200  &"["[!*iter->name], iter->name, *iter->name ? "] " : "",
201  c ? "connection" : "connector",
202  IO_StatusStr(c ? status : eIO_Unknown)));
203  if (c && c->destroy)
204  c->destroy(c);
205  assert(0);
206  }
207 }
208 
209 
210 static int/*bool*/ s_Update(SERV_ITER iter, const char* text, int code)
211 {
212  static const char server_info[] = "Server-Info-";
213  struct SDISPD_Data* data = (struct SDISPD_Data*) iter->data;
214  int/*bool*/ failure;
215 
216  if (strncasecmp(text, server_info, sizeof(server_info) - 1) == 0
217  && isdigit((unsigned char) text[sizeof(server_info) - 1])) {
218  const char* name;
219  SSERV_Info* info;
220  unsigned int d1;
221  char* s;
222  int d2;
223 
224  text += sizeof(server_info) - 1;
225  if (sscanf(text, "%u: %n", &d1, &d2) < 1 || d1 < 1)
226  return 0/*not updated*/;
227  if (iter->ismask || iter->reverse_dns) {
228  char* c;
229  if (!(s = strdup(text + d2)))
230  return 0/*not updated*/;
231  name = s;
232  while (*name && isspace((unsigned char)(*name)))
233  name++;
234  if (!*name) {
235  free(s);
236  return 0/*not updated*/;
237  }
238  for (c = s + (name - s); *c; c++) {
239  if (isspace((unsigned char)(*c)))
240  break;
241  }
242  *c++ = '\0';
243  d2 += (int)(c - s);
244  } else {
245  s = 0;
246  name = "";
247  }
248  info = SERV_ReadInfoEx(text + d2, name, 0);
249  if (s)
250  free(s);
251  if (info) {
252  if (info->time != NCBI_TIME_INFINITE)
253  info->time += iter->time; /* expiration time now */
254  if (s_AddServerInfo(data, info))
255  return 1/*updated*/;
256  free(info);
257  }
258  } else if (((failure = strncasecmp(text, HTTP_DISP_FAILURES,
259  sizeof(HTTP_DISP_FAILURES) - 1) == 0)
261  sizeof(HTTP_DISP_MESSAGES) - 1) == 0) &&
262  isspace((unsigned char) text[sizeof(HTTP_DISP_FAILURES) - 1])) {
263  assert(sizeof(HTTP_DISP_FAILURES) == sizeof(HTTP_DISP_MESSAGES));
264 #if defined(_DEBUG) && !defined(NDEBUG)
265  if (data->net_info->debug_printout) {
266  text += sizeof(HTTP_DISP_FAILURES) - 1;
267  while (*text && isspace((unsigned char)(*text)))
268  text++;
270  ("[%s] %s", data->net_info->svc/*not exact*/, text));
271  }
272 #endif /*_DEBUG && !NDEBUG*/
273  if (failure) {
274  if (code)
275  data->fail = 1;
276  return 1/*updated*/;
277  }
278  /* NB: a mere message does not constitute an update */
279  }
280 
281  return 0/*not updated*/;
282 }
283 
284 
285 static int/*bool*/ s_IsUpdateNeeded(TNCBI_Time now, struct SDISPD_Data *data)
286 {
287  double status = 0.0, total = 0.0;
288 
289  if (data->n_cand) {
290  size_t i = 0;
291  while (i < data->n_cand) {
292  const SSERV_Info* info = data->cand[i].info;
293 
294  total += fabs(info->rate);
295  if (info->time < now) {
296  if (i < --data->n_cand) {
297  memmove(data->cand + i, data->cand + i + 1,
298  sizeof(*data->cand)*(data->n_cand - i));
299  }
300  free((void*) info);
301  } else {
302  status += fabs(info->rate);
303  i++;
304  }
305  }
306  }
307  return total == 0.0 ? 1 : status/total < DISPD_STALE_RATIO_OK;
308 }
309 
310 
311 static SLB_Candidate* s_GetCandidate(void* user_data, size_t n)
312 {
313  struct SDISPD_Data* data = (struct SDISPD_Data*) user_data;
314  return n < data->n_cand ? &data->cand[n] : 0;
315 }
316 
317 
318 static SSERV_Info* s_GetNextInfo(SERV_ITER iter, HOST_INFO* host_info)
319 {
320  struct SDISPD_Data* data = (struct SDISPD_Data*) iter->data;
321  SSERV_Info* info;
322  size_t n;
323 
324  assert(data);
325  if (!data->fail && iter->n_skip < data->n_skip)
326  data->eof = 0/*false*/;
327  data->n_skip = iter->n_skip;
328 
329  if (s_IsUpdateNeeded(iter->time, data)) {
330  if (!(data->eof | data->fail))
331  s_Resolve(iter);
332  if (!data->n_cand)
333  return 0;
334  }
335 
336  for (n = 0; n < data->n_cand; n++)
337  data->cand[n].status = data->cand[n].info->rate;
339  info = (SSERV_Info*) data->cand[n].info;
340  info->rate = data->cand[n].status;
341  if (n < --data->n_cand) {
342  memmove(data->cand + n, data->cand + n + 1,
343  (data->n_cand - n) * sizeof(*data->cand));
344  }
345 
346  if (host_info)
347  *host_info = 0;
348  data->n_skip++;
349 
350  return info;
351 }
352 
353 
354 static void s_Reset(SERV_ITER iter)
355 {
356  struct SDISPD_Data* data = (struct SDISPD_Data*) iter->data;
357  if (data) {
358  data->eof = data->fail = 0/*false*/;
359  if (data->cand) {
360  size_t i;
361  assert(data->a_cand);
362  for (i = 0; i < data->n_cand; i++)
363  free((void*) data->cand[i].info);
364  data->n_cand = 0;
365  }
366  data->n_skip = iter->n_skip;
367  }
368 }
369 
370 
371 static void s_Close(SERV_ITER iter)
372 {
373  struct SDISPD_Data* data = (struct SDISPD_Data*) iter->data;
374  assert(!data->n_cand); /*s_Reset() had to be called before*/
375  if (data->cand)
376  free(data->cand);
378  free(data);
379  iter->data = 0;
380 }
381 
382 
383 /***********************************************************************
384  * EXTERNAL
385  ***********************************************************************/
386 
387 /*ARGSUSED*/
389  const SConnNetInfo* net_info,
390  SSERV_Info** info, HOST_INFO* u/*unused*/)
391 {
392  struct SDISPD_Data* data;
393 
394  if (!(data = (struct SDISPD_Data*) calloc(1, sizeof(*data))))
395  return 0;
396  iter->data = data;
397 
398  assert(net_info); /*must be called with non-NULL*/
399  data->net_info = ConnNetInfo_Clone(net_info);
400  if (!ConnNetInfo_SetupStandardArgs(data->net_info, iter->name)) {
401  s_Close(iter);
402  return 0;
403  }
404 
405  if (g_NCBI_ConnectRandomSeed == 0) {
408  }
409 
410  data->net_info->scheme = eURL_Https;
412  if (iter->stateless)
413  data->net_info->stateless = 1/*true*/;
414  if ((iter->types & fSERV_Firewall) && !data->net_info->firewall)
417  "User-Agent: NCBIServiceDispatcher/"
419 #ifdef NCBI_CXX_TOOLKIT
420  " (CXX Toolkit)"
421 #else
422  " (C Toolkit)"
423 #endif /*NCBI_CXX_TOOLKIT*/
424  "\r\n");
425  data->n_skip = iter->n_skip;
426 
427  iter->op = &s_op; /*SERV_Update() [from HTTP callback] expects*/
428  s_Resolve(iter);
429  iter->op = 0;
430 
431  if (!data->n_cand && (data->fail
432  || !(data->net_info->stateless &&
433  data->net_info->firewall))) {
434  s_Reset(iter);
435  s_Close(iter);
436  return 0;
437  }
438 
439  /* call GetNextInfo subsequently if info is actually needed */
440  if (info)
441  *info = 0;
442  return &s_op;
443 }
#define memmove(a, b, c)
static void s_Resolve(SERV_ITER iter)
Definition: ncbi_dispd.c:156
int ConnNetInfo_SetupStandardArgs(SConnNetInfo *info, const char *service)
unsigned reverse_dns
SLB_Candidate * cand
Definition: ncbi_dispd.c:76
#define HTTP_DISP_MESSAGES
Definition: ncbi_comm.h:41
#define strncasecmp
void ConnNetInfo_Destroy(SConnNetInfo *info)
unsigned ok_suppressed
#define assert(x)
Definition: srv_diag.hpp:58
Parse succeeded, retain server status.
int ConnNetInfo_ExtendUserHeader(SConnNetInfo *info, const char *header)
Regular firewall ports first, then fallback.
#define NCBI_TIME_INFINITE
Definition: ncbi_types.h:147
#define HTTP_DISP_FAILURES
Definition: ncbi_comm.h:40
size_t n_cand
Definition: ncbi_dispd.c:77
#define NCBI_DISP_VERSION
Definition: ncbi_comm.h:38
static const SSERV_VTable s_op
Definition: ncbi_dispd.c:64
const SSERV_VTable * op
EIO_Status
I/O status.
Definition: ncbi_core.h:126
const SSERV_VTable * SERV_DISPD_Open(SERV_ITER iter, const SConnNetInfo *net_info, SSERV_Info **info, HOST_INFO *u)
Definition: ncbi_dispd.c:388
EIO_Status CONN_Close(CONN conn)
Close the connection and destroy all relevant internal data.
int SERV_EqualInfo(const SSERV_Info *info1, const SSERV_Info *info2)
int g_NCBI_ConnectRandomSeed
Definition: ncbi_priv.c:51
int i
void ConnNetInfo_DeleteUserHeader(SConnNetInfo *info, const char *header)
const char svc[1]
CONNECTOR HTTP_CreateConnectorEx(const SConnNetInfo *net_info, THTTP_Flags flags, FHTTP_ParseHeader parse_header, void *user_data, FHTTP_Adjust adjust, FHTTP_Cleanup cleanup)
Create new CONNECTOR structure to hit the specified URL using HTTP with either POST / GET (or ANY) me...
size_t LB_Select(SERV_ITER iter, void *data, FGetCandidate get_candidate, double bonus)
Definition: ncbi_lb.c:62
static int s_IsUpdateNeeded(TNCBI_Time now, struct SDISPD_Data *data)
Definition: ncbi_dispd.c:285
EIO_Status CONN_Create(CONNECTOR connector, CONN *conn)
Same as CONN_CreateEx() called with 0 in the "flags" parameter.
short fail
Definition: ncbi_dispd.c:74
static void s_Close(SERV_ITER)
Definition: ncbi_dispd.c:371
everything is fine, no error occurred
Definition: ncbi_core.h:127
static SLB_Candidate * s_GetCandidate(void *user_data, size_t n)
Definition: ncbi_dispd.c:311
const char * SERV_NameOfInfo(const SSERV_Info *info)
unsigned ismask
Definition: ncbi_servicep.h:96
static int s_Adjust(SConnNetInfo *net_info, void *iter, unsigned int unused)
Definition: ncbi_dispd.c:147
static int s_AddServerInfo(struct SDISPD_Data *data, SSERV_Info *info)
Definition: ncbi_dispd.c:83
EHTTP_HeaderParse
The extended version HTTP_CreateConnectorEx() is able to track the HTTP response chain and also chang...
int SERV_Update(SERV_ITER iter, const char *text, int code)
Definition: ncbi_service.c:676
double status
Definition: ncbi_lb.h:49
#define NCBI_CXX_TOOLKIT
Definition: ncbiconf_msvc.h:16
unsigned stateless
static int s_Update(SERV_ITER, const char *, int)
Definition: ncbi_dispd.c:210
#define DISPD_LOCAL_BONUS
Definition: ncbi_dispd.c:53
Parse failed, treat as a server error.
int isdigit(Uchar c)
Definition: ncbictype.hpp:64
static SSERV_Info * s_GetNextInfo(SERV_ITER, HOST_INFO *)
Definition: ncbi_dispd.c:318
static MDB_envinfo info
Definition: mdb_load.c:37
EBFWMode firewall
char * SERV_Print(SERV_ITER iter, SConnNetInfo *net_info, int but_last)
Definition: ncbi_service.c:756
if(yy_accept[yy_current_state])
unsigned int TNCBI_Time
Definition: ncbi_types.h:145
TBSERV_TypeOnly types
Definition: ncbi_servicep.h:95
unsigned ok_down
Definition: ncbi_servicep.h:97
EIO_Status CONN_Flush(CONN conn)
Explicitly flush connection from any pending data written by CONN_Write().
EBURLScheme scheme
Definition: inftrees.h:24
int ConnNetInfo_OverrideUserHeader(SConnNetInfo *info, const char *header)
Connector will really flush on Flush()
size_t n_skip
Definition: ncbi_dispd.c:79
EBDebugPrintout debug_printout
static void s_Reset(SERV_ITER)
Definition: ncbi_dispd.c:354
unknown (most probably – fatal) error
Definition: ncbi_core.h:133
const char * IO_StatusStr(EIO_Status status)
Get the text form of an enum status value.
Definition: ncbi_core.c:56
unsigned stateless
unsigned int
A callback function used to compare two keys in a database.
Definition: types.hpp:1153
TNCBI_Time time
Definition: ncbi_servicep.h:92
const char * name
Definition: ncbi_servicep.h:90
static CS_CONNECTION * conn
Definition: get_send_data.c:18
TNCBI_Time time
static int failure
Definition: t0019.c:14
yy_size_t n
const SSERV_Info * info
Definition: ncbi_lb.h:48
int isspace(Uchar c)
Definition: ncbictype.hpp:69
static EHTTP_HeaderParse s_ParseHeader(const char *header, void *iter, int server_error)
Definition: ncbi_dispd.c:120
size_t a_cand
Definition: ncbi_dispd.c:78
static void text(MDB_val *v)
Definition: mdb_dump.c:62
#define fabs(v)
Definition: ncbi_dispd.c:46
short eof
Definition: ncbi_dispd.c:73
SSERV_Info * SERV_ReadInfoEx(const char *str, const char *name, int lazy)
#define strdup
Definition: ncbi_ansi_ext.h:70
SConnNetInfo * net_info
Definition: ncbi_dispd.c:75
TReqMethod req_method
#define CORE_LOGF_X(subcode, level, fmt_args)
Definition: ncbi_priv.h:151
SConnNetInfo * ConnNetInfo_Clone(const SConnNetInfo *info)
#define strcasecmp
Connector specification.
#define NCBI_CONNECT_SRAND_ADDEND
Definition: ncbi_priv.h:345
#define DISPD_STALE_RATIO_OK
Definition: ncbi_dispd.c:51
Modified on Sat Sep 23 14:32:30 2017 by modify_doxy.py rev. 546573