30 {
"$Id: HTTPConnect.cc 24380 2011-03-28 21:47:15Z jimg $"
84 #define CLIENT_ERR_MIN 400
85 #define CLIENT_ERR_MAX 417
89 "Unauthorized: Contact the server administrator.",
91 "Forbidden: Contact the server administrator.",
92 "Not Found: The data source or server could not be found.\n\
93 Often this means that the OPeNDAP server is missing or needs attention;\n\
94 Please contact the server administrator.",
95 "Method Not Allowed.",
97 "Proxy Authentication Required.",
102 "Precondition Failed.",
103 "Request Entity Too Large.",
104 "Request URI Too Large.",
105 "Unsupported Media Type.",
106 "Requested Range Not Satisfiable.",
107 "Expectation Failed."
110 #define SERVER_ERR_MIN 500
111 #define SERVER_ERR_MAX 505
114 "Internal Server Error.",
117 "Service Unavailable.",
119 "HTTP Version Not Supported."
125 http_status_to_string(
int status)
132 return string(
"Unknown Error: This indicates a problem with libdap++.\nPlease report this to support@opendap.org.");
139 class ParseHeader :
public unary_function<const string &, void>
147 ParseHeader() : type(
unknown_type), server(
"dods/0.0"), protocol(
"2.0")
150 void operator()(
const string &line)
154 if (name ==
"content-description") {
155 DBG2(cerr << name <<
": " << value << endl);
161 else if (name ==
"xdods-server" && server ==
"dods/0.0") {
162 DBG2(cerr << name <<
": " << value << endl);
165 else if (name ==
"xopendap-server") {
166 DBG2(cerr << name <<
": " << value << endl);
169 else if (name ==
"xdap") {
170 DBG2(cerr << name <<
": " << value << endl);
173 else if (server ==
"dods/0.0" && name ==
"server") {
174 DBG2(cerr << name <<
": " << value << endl);
177 else if (name ==
"location") {
178 DBG2(cerr << name <<
": " << value << endl);
182 && line.find(
"text/html") != string::npos) {
183 DBG2(cerr << name <<
": text/html..." << endl);
198 string get_protocol()
203 string get_location() {
225 save_raw_http_headers(
void *ptr,
size_t size,
size_t nmemb,
void *resp_hdrs)
227 DBG2(cerr <<
"Inside the header parser." << endl);
228 vector<string> *hdrs =
static_cast<vector<string> *
>(resp_hdrs);
231 string complete_line;
232 if (nmemb > 1 && *(static_cast<char*>(ptr) + size * (nmemb - 2)) ==
'\r')
233 complete_line.assign(static_cast<char *>(ptr), size * (nmemb - 2));
235 complete_line.assign(static_cast<char *>(ptr), size * (nmemb - 1));
238 if (complete_line !=
"" && complete_line.find(
"HTTP") == string::npos) {
239 DBG(cerr <<
"Header line: " << complete_line << endl);
240 hdrs->push_back(complete_line);
248 curl_debug(CURL *, curl_infotype info,
char *msg,
size_t size,
void *)
250 string message(msg, size);
254 cerr <<
"Text: " << message;
break;
255 case CURLINFO_HEADER_IN:
256 cerr <<
"Header in: " << message;
break;
257 case CURLINFO_HEADER_OUT:
258 cerr <<
"Header out: " << message;
break;
259 case CURLINFO_DATA_IN:
260 cerr <<
"Data in: " << message;
break;
261 case CURLINFO_DATA_OUT:
262 cerr <<
"Data out: " << message;
break;
264 cerr <<
"End: " << message;
break;
265 #ifdef CURLINFO_SSL_DATA_IN
266 case CURLINFO_SSL_DATA_IN:
267 cerr <<
"SSL Data in: " << message;
break;
269 #ifdef CURLINFO_SSL_DATA_OUT
270 case CURLINFO_SSL_DATA_OUT:
271 cerr <<
"SSL Data out: " << message;
break;
274 cerr <<
"Curl info: " << message;
break;
283 HTTPConnect::www_lib_init()
285 d_curl = curl_easy_init();
287 throw InternalErr(__FILE__, __LINE__,
"Could not initialize libcurl.");
293 if (!d_rcr->get_proxy_server_host().empty()) {
294 DBG(cerr <<
"Setting up a proxy server." << endl);
295 DBG(cerr <<
"Proxy host: " << d_rcr->get_proxy_server_host()
297 DBG(cerr <<
"Proxy port: " << d_rcr->get_proxy_server_port()
299 DBG(cerr <<
"Proxy pwd : " << d_rcr->get_proxy_server_userpw()
301 curl_easy_setopt(d_curl, CURLOPT_PROXY,
302 d_rcr->get_proxy_server_host().c_str());
303 curl_easy_setopt(d_curl, CURLOPT_PROXYPORT,
304 d_rcr->get_proxy_server_port());
307 #ifdef CURLOPT_PROXYAUTH
308 curl_easy_setopt(d_curl, CURLOPT_PROXYAUTH, (
long)CURLAUTH_ANY);
312 if (!d_rcr->get_proxy_server_userpw().empty())
313 curl_easy_setopt(d_curl, CURLOPT_PROXYUSERPWD,
314 d_rcr->get_proxy_server_userpw().c_str());
317 curl_easy_setopt(d_curl, CURLOPT_ERRORBUFFER, d_error_buffer);
320 curl_easy_setopt(d_curl, CURLOPT_FAILONERROR, 0);
325 curl_easy_setopt(d_curl, CURLOPT_HTTPAUTH, (
long)CURLAUTH_ANY);
327 curl_easy_setopt(d_curl, CURLOPT_NOPROGRESS, 1);
328 curl_easy_setopt(d_curl, CURLOPT_NOSIGNAL, 1);
329 curl_easy_setopt(d_curl, CURLOPT_HEADERFUNCTION, save_raw_http_headers);
334 curl_easy_setopt(d_curl, CURLOPT_FOLLOWLOCATION, 1);
335 curl_easy_setopt(d_curl, CURLOPT_MAXREDIRS, 5);
338 if (!d_rcr->get_validate_ssl() == 0) {
339 curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYPEER, 0);
340 curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYHOST, 0);
347 if (!d_cookie_jar.empty()) {
348 DBG(cerr <<
"Setting the cookie jar to: " << d_cookie_jar << endl);
349 curl_easy_setopt(d_curl, CURLOPT_COOKIEJAR, d_cookie_jar.c_str());
350 curl_easy_setopt(d_curl, CURLOPT_COOKIESESSION, 1);
354 cerr <<
"Curl version: " << curl_version() << endl;
355 curl_easy_setopt(d_curl, CURLOPT_VERBOSE, 1);
356 curl_easy_setopt(d_curl, CURLOPT_DEBUGFUNCTION, curl_debug);
363 class BuildHeaders :
public unary_function<const string &, void>
365 struct curl_slist *d_cl;
368 BuildHeaders() : d_cl(0)
371 void operator()(
const string &header)
373 DBG(cerr <<
"Adding '" << header.c_str() <<
"' to the header list."
375 d_cl = curl_slist_append(d_cl, header.c_str());
378 struct curl_slist *get_headers()
399 HTTPConnect::read_url(
const string &url, FILE *stream,
400 vector<string> *resp_hdrs,
401 const vector<string> *headers)
403 curl_easy_setopt(d_curl, CURLOPT_URL, url.c_str());
413 curl_easy_setopt(d_curl, CURLOPT_FILE, stream);
414 curl_easy_setopt(d_curl, CURLOPT_WRITEFUNCTION, &fwrite);
416 curl_easy_setopt(d_curl, CURLOPT_FILE, stream);
419 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
420 ostream_iterator<string>(cerr,
"\n")));
422 BuildHeaders req_hdrs;
423 req_hdrs = for_each(d_request_headers.begin(), d_request_headers.end(),
426 req_hdrs = for_each(headers->begin(), headers->end(), req_hdrs);
427 curl_easy_setopt(d_curl, CURLOPT_HTTPHEADER, req_hdrs.get_headers());
430 bool temporary_proxy =
false;
431 if ((temporary_proxy = url_uses_no_proxy_for(url))) {
432 DBG(cerr <<
"Suppress proxy for url: " << url << endl);
433 curl_easy_setopt(d_curl, CURLOPT_PROXY, 0);
436 string::size_type at_sign = url.find(
'@');
440 if (at_sign != url.npos)
441 d_upstring = url.substr(7, at_sign - 7);
443 if (!d_upstring.empty())
444 curl_easy_setopt(d_curl, CURLOPT_USERPWD, d_upstring.c_str());
449 curl_easy_setopt(d_curl, CURLOPT_WRITEHEADER, resp_hdrs);
451 CURLcode res = curl_easy_perform(d_curl);
454 curl_slist_free_all(req_hdrs.get_headers());
455 curl_easy_setopt(d_curl, CURLOPT_HTTPHEADER, 0);
458 if (temporary_proxy && !d_rcr->get_proxy_server_host().empty())
459 curl_easy_setopt(d_curl, CURLOPT_PROXY,
460 d_rcr->get_proxy_server_host().c_str());
463 throw Error(d_error_buffer);
466 res = curl_easy_getinfo(d_curl, CURLINFO_HTTP_CODE, &status);
468 throw Error(d_error_buffer);
477 HTTPConnect::url_uses_proxy_for(
const string &url)
throw()
479 if (d_rcr->is_proxy_for_used()) {
480 Regex host_regex(d_rcr->get_proxy_for_regexp().c_str());
481 int index = 0, matchlen;
482 return host_regex.search(url.c_str(), url.size(), matchlen, index) != -1;
492 HTTPConnect::url_uses_no_proxy_for(
const string &url)
throw()
494 return d_rcr->is_no_proxy_for_used()
495 && url.find(d_rcr->get_no_proxy_for_host()) != string::npos;
506 HTTPConnect::HTTPConnect(
RCReader *rcr) : d_username(
""), d_password(
""),
508 d_dap_client_protocol_major(2),
509 d_dap_client_protocol_minor(0)
519 d_request_headers.push_back(
string(
"Pragma:"));
520 string user_agent = string(
"User-Agent: ") + string(
CNAME)
521 + string(
"/") + string(
CVER);
522 d_request_headers.push_back(user_agent);
523 if (d_accept_deflate)
524 d_request_headers.push_back(
string(
"Accept-Encoding: deflate, gzip, compress"));
533 DBG2(cerr <<
"Cache object created (" << hex << d_http_cache << dec
552 DBG2(cerr <<
"Entering the HTTPConnect dtor" << endl);
554 curl_easy_cleanup(d_curl);
556 DBG2(cerr <<
"Leaving the HTTPConnect dtor" << endl);
575 cout <<
"GET " << url <<
" HTTP/1.0" << endl;
581 stream = caching_fetch_url(url);
584 stream = plain_fetch_url(url);
589 ss <<
"HTTP/1.0 " << stream->
get_status() <<
" -" << endl;
590 for (
size_t i = 0; i < stream->
get_headers()->size(); i++) {
602 cout << endl << endl;
606 if (parser.get_location() !=
"" &&
607 url.substr(0,url.find(
"?",0)).compare(parser.get_location().substr(0,url.find(
"?",0))) != 0) {
612 stream->
set_type(parser.get_object_type());
630 get_tempfile_template(
const string &file_template)
637 Regex directory(
"[-a-zA-Z0-9_:\\]*");
642 if (c && directory.match(c.c_str(), c.length()) && (access(c.c_str(), 6) == 0))
643 goto valid_temp_directory;
646 if (c && directory.match(c.c_str(), c.length()) && (access(c.c_str(), 6) == 0))
647 goto valid_temp_directory;
652 if (c && directory.match(c.c_str(), c.length()) && (access(c.c_str(), 6) == 0))
653 goto valid_temp_directory;
655 #else // Unix/Linux/OSX has another...
657 Regex directory(
"[-a-zA-Z0-9_/]*");
659 c = getenv(
"TMPDIR");
660 if (directory.match(c.c_str(), c.length()) && (access(c.c_str(), W_OK | R_OK) == 0))
661 goto valid_temp_directory;
666 if (access(P_tmpdir, W_OK | R_OK) == 0) {
668 goto valid_temp_directory;
674 if (directory.match(c.c_str(), c.length()) && (access(c.c_str(), W_OK | R_OK) == 0))
675 goto valid_temp_directory;
682 valid_temp_directory:
685 c +=
"\\" + file_template;
687 c +=
"/" + file_template;
714 string dods_temp = get_tempfile_template((
string)
"dodsXXXXXX");
716 vector<char> pathname(dods_temp.length() + 1);
718 strncpy(&pathname[0], dods_temp.c_str(), dods_temp.length());
720 DBG(cerr <<
"pathanme: " << &pathname[0] <<
" (" << dods_temp.length() + 1 <<
")" << endl);
723 #if defined(WIN32) || defined(TEST_WIN32_TEMPS)
724 stream = fopen(_mktemp(&pathname[0]),
"w+b");
728 stream = fdopen(mkstemp(&pathname[0]),
"w+");
733 "Failed to open a temporary file for the data values ("
737 dods_temp = &pathname[0];
749 res = unlink(name.c_str());
776 HTTPConnect::caching_fetch_url(
const string &url)
778 DBG(cerr <<
"Is this URL (" << url <<
") in the cache?... ");
780 vector<string> *headers =
new vector<string>;
785 DBGN(cerr <<
"no; getting response and caching." << endl);
786 delete headers; headers = 0;
787 time_t now = time(0);
788 HTTPResponse *rs = plain_fetch_url(url);
789 d_http_cache->
cache_response(url, now, *(rs->get_headers()), rs->get_stream());
794 DBGN(cerr <<
"yes... ");
797 DBGN(cerr <<
"and it's valid; using cached response." << endl);
798 HTTPCacheResponse *crs =
new HTTPCacheResponse(s, 200, headers, file_name, d_http_cache);
802 DBGN(cerr <<
"but it's not valid; validating... ");
809 time_t now = time(0);
813 http_status = read_url(url, body, headers, &cond_hdrs);
822 switch (http_status) {
824 DBGN(cerr <<
"read a new response; caching." << endl);
827 HTTPResponse *rs =
new HTTPResponse(body, http_status, headers, dods_temp);
833 DBGN(cerr <<
"cached response valid; updating." << endl);
839 HTTPCacheResponse *crs =
new HTTPCacheResponse(hs, 304, headers, file_name, d_http_cache);
845 if (http_status >= 400) {
846 delete headers; headers = 0;
847 string msg =
"Error while reading the URL: ";
850 +=
".\nThe OPeNDAP server returned the following message:\n";
851 msg += http_status_to_string(http_status);
855 delete headers; headers = 0;
856 throw InternalErr(__FILE__, __LINE__,
857 "Bad response from the HTTP server: " +
long_to_string(http_status));
864 throw InternalErr(__FILE__, __LINE__,
"Should never get here");
879 HTTPConnect::plain_fetch_url(
const string &url)
881 DBG(cerr <<
"Getting URL: " << url << endl);
884 vector<string> *resp_hdrs =
new vector<string>;
888 status = read_url(url, stream, resp_hdrs);
891 string msg =
"Error while reading the URL: ";
893 msg +=
".\nThe OPeNDAP server returned the following message:\n";
894 msg += http_status_to_string(status);
907 return new HTTPResponse(stream, status, resp_hdrs, dods_temp);
926 if (d_accept_deflate) {
927 if (find(d_request_headers.begin(), d_request_headers.end(),
928 "Accept-Encoding: deflate, gzip, compress") == d_request_headers.end())
929 d_request_headers.push_back(
string(
"Accept-Encoding: deflate, gzip, compress"));
930 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
931 ostream_iterator<string>(cerr,
"\n")));
934 vector<string>::iterator i;
935 i = remove_if(d_request_headers.begin(), d_request_headers.end(),
936 bind2nd(equal_to<string>(),
937 string(
"Accept-Encoding: deflate, gzip, compress")));
938 d_request_headers.erase(i, d_request_headers.end());
943 class HeaderMatch :
public unary_function<const string &, bool> {
944 const string &d_header;
946 HeaderMatch(
const string &header) : d_header(header) {}
947 bool operator()(
const string &arg) {
return arg.find(d_header) == 0; }
962 vector<string>::iterator i;
963 i = find_if(d_request_headers.begin(), d_request_headers.end(),
964 HeaderMatch(
"XDAP-Accept:"));
965 if (i != d_request_headers.end())
966 d_request_headers.erase(i);
969 d_dap_client_protocol_major = major;
970 d_dap_client_protocol_minor = minor;
971 ostringstream xdap_accept;
972 xdap_accept <<
"XDAP-Accept: " << major <<
"." << minor;
974 d_request_headers.push_back(xdap_accept.str());
976 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
977 ostream_iterator<string>(cerr,
"\n")));
1005 d_upstring = u +
":" + p;
vector< string > get_conditional_request_headers(const string &url)
bool is_cache_enabled() const
virtual int get_status() const
void set_cache_enabled(bool mode)
bool is_url_valid(const string &url)
void set_credentials(const string &u, const string &p)
static HTTPCache * instance(const string &cache_root, bool force=false)
void set_max_size(unsigned long size)
int get_ignore_expires() const
virtual void set_type(ObjectType o)
FILE * get_cached_response(const string &url, vector< string > &headers, string &cacheName)
string get_cookie_jar() const
ObjectType
The type of object in the stream coming from the data server.
HTTPResponse * fetch_url(const string &url)
virtual void set_version(const string &v)
int get_default_expires() const
A class for software fault reporting.
void parse_mime_header(const string &header, string &name, string &value)
virtual void set_protocol(const string &p)
unsigned int get_max_cached_obj() const
int get_max_cache_size() const
ObjectType get_description_type(const string &value)
void update_response(const string &url, time_t request_time, const vector< string > &headers)
void close_temp(FILE *s, const string &name)
string get_temp_file(FILE *&stream)
bool cache_response(const string &url, time_t request_time, const vector< string > &headers, const FILE *body)
void set_accept_deflate(bool defalte)
string long_to_string(long val, int base)
void set_always_validate(bool validate)
void set_xdap_protocol(int major, int minor)
void set_default_expiration(int exp_time)
bool get_use_cache() const
int get_always_validate() const
virtual vector< string > * get_headers() const
void release_cached_response(FILE *response)
void set_expire_ignored(bool mode)
string get_dods_cache_root() const
void set_max_entry_size(unsigned long size)