35 #include <sys/types.h>
61 #define MKDIR(a,b) _mkdir((a))
62 #define REMOVE(a) do { \
63 int s = remove((a)); \
65 throw InternalErr(__FILE__, __LINE__, "Coule not remove file: " + long_to_string(s)); \
67 #define MKSTEMP(a) _open(_mktemp((a)),_O_CREAT,_S_IREAD|_S_IWRITE)
68 #define DIR_SEPARATOR_CHAR '\\'
69 #define DIR_SEPARATOR_STR "\\"
71 #define MKDIR(a,b) mkdir((a), (b))
72 #define REMOVE(a) remove((a))
73 #define MKSTEMP(a) mkstemp((a))
74 #define DIR_SEPARATOR_CHAR '/'
75 #define DIR_SEPARATOR_STR "/"
78 #define CACHE_META ".meta"
79 #define CACHE_INDEX ".index"
80 #define CACHE_EMPTY_ETAG "@cache@"
82 #define NO_LM_EXPIRATION 24*3600 // 24 hours
83 #define MAX_LM_EXPIRATION 48*3600 // Max expiration from LM
88 #define LM_EXPIRATION(t) (min((MAX_LM_EXPIRATION), static_cast<int>((t) / 10)))
105 for (
const char *ptr = url.c_str(); *ptr; ptr++)
111 HTTPCacheTable::HTTPCacheTable(
const string &cache_root,
int block_size) :
112 d_cache_root(cache_root), d_block_size(block_size), d_current_size(0), d_new_entries(0)
120 d_cache_table[i] = 0;
131 DBG2(cerr <<
"Deleting CacheEntry: " << e << endl);
141 for_each(cp->begin(), cp->end(), delete_cache_entry);
144 delete get_cache_table()[i];
145 get_cache_table()[i] = 0;
149 delete[] d_cache_table;
159 class DeleteExpired :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
165 d_time(t), d_table(table) {
170 void operator()(HTTPCacheTable::CacheEntry *&e) {
171 if (e && !e->readers && (e->freshness_lifetime
172 < (e->corrected_initial_age + (d_time - e->response_time)))) {
173 DBG(cerr <<
"Deleting expired cache entry: " << e->url << endl);
174 d_table.remove_cache_entry(e);
186 for_each(slot->begin(), slot->end(), DeleteExpired(*
this, time));
187 slot->erase(
remove(slot->begin(), slot->end(),
199 class DeleteByHits :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
205 d_table(table), d_hits(hits) {
208 void operator()(HTTPCacheTable::CacheEntry *&e) {
209 if (e && !e->readers && e->hits <= d_hits) {
210 DBG(cerr <<
"Deleting cache entry: " << e->url << endl);
211 d_table.remove_cache_entry(e);
220 if (get_cache_table()[cnt]) {
222 for_each(slot->begin(), slot->end(), DeleteByHits(*
this, hits));
223 slot->erase(
remove(slot->begin(), slot->end(),
235 class DeleteBySize :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
241 d_table(table), d_size(size) {
244 void operator()(HTTPCacheTable::CacheEntry *&e) {
245 if (e && !e->readers && e->size > d_size) {
246 DBG(cerr <<
"Deleting cache entry: " << e->url << endl);
247 d_table.remove_cache_entry(e);
255 if (get_cache_table()[cnt]) {
257 for_each(slot->begin(), slot->end(), DeleteBySize(*
this, size));
258 slot->erase(
remove(slot->begin(), slot->end(),
284 return (
REMOVE(d_cache_index.c_str()) == 0);
298 FILE *fp = fopen(d_cache_index.c_str(),
"r");
306 while (!feof(fp) && fgets(line, 1024, fp)) {
308 DBG2(cerr << line << endl);
311 int res = fclose(fp) ;
313 DBG(cerr <<
"HTTPCache::cache_index_read - Failed to close " << (
void *)fp << endl);
333 istringstream iss(line);
335 iss >> entry->cachename;
342 iss >> entry->expires;
348 iss >> entry->freshness_lifetime;
349 iss >> entry->response_time;
350 iss >> entry->corrected_initial_age;
352 iss >> entry->must_revalidate;
359 class WriteOneCacheEntry :
360 public unary_function<HTTPCacheTable::CacheEntry *, void>
366 WriteOneCacheEntry(FILE *fp) : d_fp(fp)
369 void operator()(HTTPCacheTable::CacheEntry *e)
371 if (e && fprintf(d_fp,
372 "%s %s %s %ld %ld %ld %c %d %d %ld %ld %ld %c\r\n",
374 e->cachename.c_str(),
379 e->range ?
'1' :
'0',
382 (long)(e->freshness_lifetime),
383 (long)(e->response_time),
384 (long)(e->corrected_initial_age),
385 e->must_revalidate ?
'1' :
'0') < 0)
386 throw Error(
"Cache Index. Error writing cache index\n");
402 DBG(cerr <<
"Cache Index. Writing index " << d_cache_index << endl);
406 if ((fp = fopen(d_cache_index.c_str(),
"wb")) == NULL) {
407 throw Error(
string(
"Cache Index. Can't open `") + d_cache_index
408 +
string(
"' for writing"));
417 for_each(cp->begin(), cp->end(), WriteOneCacheEntry(fp));
421 int res = fclose(fp);
423 DBG(cerr <<
"HTTPCache::cache_index_write - Failed to close "
424 << (
void *)fp << endl);
447 struct stat stat_info;
450 path << d_cache_root << hash;
451 string p = path.str();
453 if (stat(p.c_str(), &stat_info) == -1) {
454 DBG2(cerr <<
"Cache....... Create dir " << p << endl);
455 if (
MKDIR(p.c_str(), 0777) < 0) {
456 DBG2(cerr <<
"Cache....... Can't create..." << endl);
457 throw Error(
"Could not create cache slot to hold response! Check the write permissions on your disk cache directory. Cache root: " + d_cache_root +
".");
461 DBG2(cerr <<
"Cache....... Directory " << p <<
" already exists"
487 hash_dir +=
"\\dodsXXXXXX";
489 hash_dir +=
"/dodsXXXXXX";
494 vector<char> templat(hash_dir.size() + 1);
495 strncpy(&templat[0], hash_dir.c_str(), hash_dir.size() + 1);
509 throw Error(
"The HTTP Cache could not create a file to hold the response; it will not be cached.");
512 entry->cachename = &templat[0];
520 entry_disk_space(
int size,
unsigned int block_size)
522 unsigned int num_of_blocks = (size + block_size) / block_size;
524 DBG(cerr <<
"size: " << size <<
", block_size: " << block_size
525 <<
", num_of_blocks: " << num_of_blocks << endl);
527 return num_of_blocks * block_size;
542 int hash = entry->hash;
544 if (!d_cache_table[hash])
547 d_cache_table[hash]->push_back(entry);
549 DBG(cerr <<
"add_entry_to_cache_table, current_size: " << d_current_size
550 <<
", entry->size: " << entry->size <<
", block size: " << d_block_size
553 d_current_size += entry_disk_space(entry->size, d_block_size);
555 DBG(cerr <<
"add_entry_to_cache_table, current_size: " << d_current_size << endl);
564 HTTPCacheTable::get_locked_entry_from_cache_table(
const string &url)
566 return get_locked_entry_from_cache_table(
get_hash(url), url);
577 HTTPCacheTable::get_locked_entry_from_cache_table(
int hash,
const string &url)
579 DBG(cerr <<
"url: " << url <<
"; hash: " << hash << endl);
580 DBG(cerr <<
"d_cache_table: " << hex << d_cache_table << dec << endl);
581 if (d_cache_table[hash]) {
586 if ((*i) && (*i)->url == url) {
587 (*i)->lock_read_response();
602 HTTPCacheTable::CacheEntry *
606 if (d_cache_table[hash]) {
611 if ((*i) && (*i)->url == url) {
612 (*i)->lock_write_response();
634 throw InternalErr(__FILE__, __LINE__,
"Tried to delete a cache entry that is in use.");
636 REMOVE(entry->cachename.c_str());
641 unsigned int eds = entry_disk_space(entry->size,
get_block_size());
649 class DeleteCacheEntry:
public unary_function<HTTPCacheTable::CacheEntry *&, void>
656 : d_url(url), d_cache_table(c)
659 void operator()(HTTPCacheTable::CacheEntry *&e)
661 if (e && e->url == d_url) {
662 e->lock_write_response();
663 d_cache_table->remove_cache_entry(e);
664 e->unlock_write_response();
680 if (d_cache_table[hash]) {
682 for_each(cp->begin(), cp->end(), DeleteCacheEntry(
this, url));
690 class DeleteUnlockedCacheEntry:
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
698 void operator()(HTTPCacheTable::CacheEntry *&e)
701 d_table.remove_cache_entry(e);
715 for_each(slot->begin(), slot->end(), DeleteUnlockedCacheEntry(*
this));
739 entry->response_time = time(NULL);
740 time_t apparent_age = max(0, static_cast<int>(entry->response_time - entry->date));
741 time_t corrected_received_age = max(apparent_age, entry->age);
742 time_t response_delay = entry->response_time - request_time;
743 entry->corrected_initial_age = corrected_received_age + response_delay;
748 time_t freshness_lifetime = entry->max_age;
749 if (freshness_lifetime < 0) {
750 if (entry->expires < 0) {
752 freshness_lifetime = default_expiration;
759 freshness_lifetime = entry->expires - entry->date;
762 entry->freshness_lifetime = max(0, static_cast<int>(freshness_lifetime));
764 DBG2(cerr <<
"Cache....... Received Age " << entry->age
765 <<
", corrected " << entry->corrected_initial_age
766 <<
", freshness lifetime " << entry->freshness_lifetime << endl);
781 const vector<string> &headers)
783 vector<string>::const_iterator i;
784 for (i = headers.begin(); i != headers.end(); ++i) {
789 string::size_type colon = (*i).find(
':');
792 if (colon == string::npos)
795 string header = (*i).substr(0, (*i).find(
':'));
796 string value = (*i).substr((*i).find(
": ") + 2);
797 DBG2(cerr <<
"Header: " << header << endl);
DBG2(cerr <<
"Value: " << value << endl);
799 if (header ==
"ETag") {
802 else if (header ==
"Last-Modified") {
805 else if (header ==
"Expires") {
808 else if (header ==
"Date") {
811 else if (header ==
"Age") {
814 else if (header ==
"Content-Length") {
815 unsigned long clength = strtoul(value.c_str(), 0, 0);
816 if (clength > max_entry_size)
819 else if (header ==
"Cache-Control") {
823 if (value ==
"no-cache" || value ==
"no-store")
828 else if (value ==
"must-revalidate")
829 entry->must_revalidate =
true;
830 else if (value.find(
"max-age") != string::npos) {
831 string max_age = value.substr(value.find(
"=" + 1));
843 d_locked_entries[body] = entry;
850 throw InternalErr(
"There is no cache entry for the response given.");
852 d_locked_entries.erase(body);
855 if (entry->readers < 0)
856 throw InternalErr(
"An unlocked entry was released");
860 return !d_locked_entries.empty();
void remove_cache_entry(HTTPCacheTable::CacheEntry *entry)
const int CACHE_TABLE_SIZE
time_t parse_time(const char *str, bool expand)
void create_location(CacheEntry *entry)
void unlock_read_response()
void parse_headers(HTTPCacheTable::CacheEntry *entry, unsigned long max_entry_size, const vector< string > &headers)
void increment_new_entries()
void add_entry_to_cache_table(CacheEntry *entry)
vector< CacheEntry * > CacheEntries
void calculate_time(HTTPCacheTable::CacheEntry *entry, int default_expiration, time_t request_time)
void delete_by_size(unsigned int size)
CacheEntries::iterator CacheEntriesIter
void set_no_cache(bool state)
A class for software fault reporting.
void delete_all_entries()
void set_current_size(unsigned long sz)
int get_hash(const string &url)
void bind_entry_to_data(CacheEntry *entry, FILE *body)
void delete_expired_entries(time_t time=0)
unsigned long get_current_size() const
CacheEntry * cache_index_parse_line(const char *line)
unsigned int get_block_size() const
void remove_entry_from_cache_table(const string &url)
CacheEntry * get_write_locked_entry_from_cache_table(const string &url)
void delete_by_hits(int hits)
A class for error processing.
string create_hash_directory(int hash)
bool cache_index_delete()
void uncouple_entry_from_data(FILE *body)
bool is_locked_read_responses()