diff options
Diffstat (limited to 'activesupport')
| -rw-r--r-- | activesupport/CHANGELOG | 6 | ||||
| -rw-r--r-- | activesupport/lib/active_support.rb | 3 | ||||
| -rw-r--r-- | activesupport/lib/active_support/cache.rb | 121 | ||||
| -rw-r--r-- | activesupport/lib/active_support/cache/compressed_mem_cache_store.rb | 15 | ||||
| -rw-r--r-- | activesupport/lib/active_support/cache/drb_store.rb | 15 | ||||
| -rw-r--r-- | activesupport/lib/active_support/cache/file_store.rb | 65 | ||||
| -rw-r--r-- | activesupport/lib/active_support/cache/mem_cache_store.rb | 51 | ||||
| -rw-r--r-- | activesupport/lib/active_support/cache/memory_store.rb | 29 | ||||
| -rw-r--r-- | activesupport/lib/active_support/core_ext/date/conversions.rb | 1 | ||||
| -rw-r--r-- | activesupport/lib/active_support/core_ext/hash/conversions.rb | 2 | ||||
| -rw-r--r-- | activesupport/lib/active_support/core_ext/time/conversions.rb | 1 | ||||
| -rw-r--r-- | activesupport/lib/active_support/gzip.rb | 22 | ||||
| -rw-r--r-- | activesupport/lib/active_support/vendor.rb | 6 | ||||
| -rw-r--r-- | activesupport/lib/active_support/vendor/memcache-client-1.5.0/memcache.rb | 832 | ||||
| -rw-r--r-- | activesupport/test/caching_test.rb | 27 |
15 files changed, 1196 insertions, 0 deletions
diff --git a/activesupport/CHANGELOG b/activesupport/CHANGELOG index e8235fb4a1..16296a579f 100644 --- a/activesupport/CHANGELOG +++ b/activesupport/CHANGELOG @@ -1,5 +1,11 @@ *SVN* +* Added ActiveSupport::Gzip.decompress/compress(source) as an easy wrapper for Zlib [Tobias Luetke] + +* Included MemCache-Client to make the improved ActiveSupport::Cache::MemCacheStore work out of the box [Bob Cottrell, Eric Hodel] + +* Added ActiveSupport::Cache::* framework as an extraction from ActionController::Caching::Fragments::* [DHH] + * Fixed String#titleize to work for strings with 's too #10571 [trek] * Changed the implementation of Enumerable#group_by to use a double array approach instead of a hash such that the insert order is honored [DHH/Marcel] diff --git a/activesupport/lib/active_support.rb b/activesupport/lib/active_support.rb index 33e6fe97bf..c459511dc8 100644 --- a/activesupport/lib/active_support.rb +++ b/activesupport/lib/active_support.rb @@ -32,6 +32,9 @@ require 'active_support/core_ext' require 'active_support/clean_logger' require 'active_support/buffered_logger' +require 'active_support/gzip' +require 'active_support/cache' + require 'active_support/dependencies' require 'active_support/deprecation' diff --git a/activesupport/lib/active_support/cache.rb b/activesupport/lib/active_support/cache.rb new file mode 100644 index 0000000000..8252ada032 --- /dev/null +++ b/activesupport/lib/active_support/cache.rb @@ -0,0 +1,121 @@ +module ActiveSupport + module Cache + def self.lookup_store(*store_option) + store, *parameters = *([ store_option ].flatten) + + case store + when Symbol + store_class_name = (store == :drb_store ? "DRbStore" : store.to_s.camelize) + store_class = ActiveSupport::Cache.const_get(store_class_name) + store_class.new(*parameters) + when nil + ActiveSupport::Cache::MemoryStore.new + else + store + end + end + + def self.expand_cache_key(key, namespace = nil) + expanded_cache_key = namespace ? "#{namespace}/" : "" + + if ENV["RAILS_CACHE_ID"] || ENV["RAILS_APP_VERSION"] + expanded_cache_key << "#{ENV["RAILS_CACHE_ID"] || ENV["RAILS_APP_VERSION"]}/" + end + + expanded_cache_key << case + when key.respond_to?(:cache_key) + key.cache_key + when key.is_a?(Array) + key.collect { |element| expand_cache_key(element) }.to_param + when key.respond_to?(:to_param) + key.to_param + end + + expanded_cache_key + end + + + class Store + cattr_accessor :logger + + def initialize + end + + def threadsafe! + @mutex = Mutex.new + self.class.send :include, ThreadSafety + self + end + + def fetch(key, options = nil) + @logger_off = true + if value = read(key, options) + @logger_off = false + log("hit", key, options) + value + elsif block_given? + @logger_off = false + log("miss", key, options) + + value = nil + seconds = Benchmark.realtime { value = yield } + + @logger_off = true + write(key, value, options) + @logger_off = false + + log("write (will save #{'%.5f' % seconds})", key, nil) + + value + end + end + + def read(key, options = nil) + log("read", key, options) + end + + def write(key, value, options = nil) + log("write", key, options) + end + + def delete(key, options = nil) + log("delete", key, options) + end + + def delete_matched(matcher, options = nil) + log("delete matched", matcher.inspect, options) + end + + + private + def log(operation, key, options) + logger.debug("Cache #{operation}: #{key}#{options ? " (#{options.inspect})" : ""}") if logger && !@logger_off + end + end + + + module ThreadSafety #:nodoc: + def read(key, options = nil) #:nodoc: + @mutex.synchronize { super } + end + + def write(key, value, options = nil) #:nodoc: + @mutex.synchronize { super } + end + + def delete(key, options = nil) #:nodoc: + @mutex.synchronize { super } + end + + def delete_matched(matcher, options = nil) #:nodoc: + @mutex.synchronize { super } + end + end + end +end + +require 'active_support/cache/file_store' +require 'active_support/cache/memory_store' +require 'active_support/cache/drb_store' +require 'active_support/cache/mem_cache_store' +require 'active_support/cache/compressed_mem_cache_store'
\ No newline at end of file diff --git a/activesupport/lib/active_support/cache/compressed_mem_cache_store.rb b/activesupport/lib/active_support/cache/compressed_mem_cache_store.rb new file mode 100644 index 0000000000..9470ac9f66 --- /dev/null +++ b/activesupport/lib/active_support/cache/compressed_mem_cache_store.rb @@ -0,0 +1,15 @@ +module ActiveSupport + module Cache + class CompressedMemCacheStore < MemCacheStore + def read(name, options = {}) + if value = super(name, options.merge(:raw => true)) + Marshal.load(ActiveSupport::Gzip.decompress(value)) + end + end + + def write(name, value, options = {}) + super(name, ActiveSupport::Gzip.compress(Marshal.dump(value)), options.merge(:raw => true)) + end + end + end +end diff --git a/activesupport/lib/active_support/cache/drb_store.rb b/activesupport/lib/active_support/cache/drb_store.rb new file mode 100644 index 0000000000..b80c2ee4d5 --- /dev/null +++ b/activesupport/lib/active_support/cache/drb_store.rb @@ -0,0 +1,15 @@ +require 'drb' + +module ActiveSupport + module Cache + class DRbStore < MemoryStore #:nodoc: + attr_reader :address + + def initialize(address = 'druby://localhost:9192') + super() + @address = address + @data = DRbObject.new(nil, address) + end + end + end +end
\ No newline at end of file diff --git a/activesupport/lib/active_support/cache/file_store.rb b/activesupport/lib/active_support/cache/file_store.rb new file mode 100644 index 0000000000..88f9ac19db --- /dev/null +++ b/activesupport/lib/active_support/cache/file_store.rb @@ -0,0 +1,65 @@ +module ActiveSupport + module Cache + class FileStore < Store + attr_reader :cache_path + + def initialize(cache_path) + @cache_path = cache_path + end + + def read(name, options = nil) + super + File.open(real_file_path(name), 'rb') { |f| f.read } rescue nil + end + + def write(name, value, options = nil) + super + ensure_cache_path(File.dirname(real_file_path(name))) + File.open(real_file_path(name), "wb+") { |f| f.write(value) } + rescue => e + RAILS_DEFAULT_LOGGER.error "Couldn't create cache directory: #{name} (#{e.message})" if RAILS_DEFAULT_LOGGER + end + + def delete(name, options) + super + File.delete(real_file_path(name)) + rescue SystemCallError => e + # If there's no cache, then there's nothing to complain about + end + + def delete_matched(matcher, options) + super + search_dir(@cache_path) do |f| + if f =~ matcher + begin + File.delete(f) + rescue SystemCallError => e + # If there's no cache, then there's nothing to complain about + end + end + end + end + + private + def real_file_path(name) + '%s/%s.cache' % [@cache_path, name.gsub('?', '.').gsub(':', '.')] + end + + def ensure_cache_path(path) + FileUtils.makedirs(path) unless File.exists?(path) + end + + def search_dir(dir, &callback) + Dir.foreach(dir) do |d| + next if d == "." || d == ".." + name = File.join(dir, d) + if File.directory?(name) + search_dir(name, &callback) + else + callback.call name + end + end + end + end + end +end
\ No newline at end of file diff --git a/activesupport/lib/active_support/cache/mem_cache_store.rb b/activesupport/lib/active_support/cache/mem_cache_store.rb new file mode 100644 index 0000000000..5820d15cc5 --- /dev/null +++ b/activesupport/lib/active_support/cache/mem_cache_store.rb @@ -0,0 +1,51 @@ +require 'memcache' + +module ActiveSupport + module Cache + class MemCacheStore < Store + attr_reader :addresses + + def initialize(*addresses) + addresses = addresses.flatten + addresses = ["localhost"] if addresses.empty? + @addresses = addresses + @data = MemCache.new(*addresses) + end + + def read(key, options = nil) + super + @data.get(key, raw?(options)) + rescue MemCache::MemCacheError + nil + end + + def write(key, value, options = nil) + super + @data.set(key, value, expires_in(options), raw?(options)) + rescue MemCache::MemCacheError + nil + end + + def delete(key, options = nil) + super + @data.delete(key, expires_in(options)) + rescue MemCache::MemCacheError + nil + end + + def delete_matched(matcher, options = nil) + super + raise "Not supported by Memcache" + end + + private + def expires_in(options) + (options && options[:expires_in]) || 0 + end + + def raw?(options) + options && options[:raw] + end + end + end +end diff --git a/activesupport/lib/active_support/cache/memory_store.rb b/activesupport/lib/active_support/cache/memory_store.rb new file mode 100644 index 0000000000..e0aba6b19a --- /dev/null +++ b/activesupport/lib/active_support/cache/memory_store.rb @@ -0,0 +1,29 @@ +module ActiveSupport + module Cache + class MemoryStore < Store + def initialize + @data = {} + end + + def read(name, options = nil) + super + @data[name] + end + + def write(name, value, options = nil) + super + @data[name] = value + end + + def delete(name, options = nil) + super + @data.delete(name) + end + + def delete_matched(matcher, options = nil) + super + @data.delete_if { |k,v| k =~ matcher } + end + end + end +end
\ No newline at end of file diff --git a/activesupport/lib/active_support/core_ext/date/conversions.rb b/activesupport/lib/active_support/core_ext/date/conversions.rb index f34d860117..684775c5e3 100644 --- a/activesupport/lib/active_support/core_ext/date/conversions.rb +++ b/activesupport/lib/active_support/core_ext/date/conversions.rb @@ -7,6 +7,7 @@ module ActiveSupport #:nodoc: :short => "%e %b", :long => "%B %e, %Y", :db => "%Y-%m-%d", + :number => "%Y%m%d", :long_ordinal => lambda { |date| date.strftime("%B #{date.day.ordinalize}, %Y") }, # => "April 25th, 2007" :rfc822 => "%e %b %Y" } diff --git a/activesupport/lib/active_support/core_ext/hash/conversions.rb b/activesupport/lib/active_support/core_ext/hash/conversions.rb index a758c3454b..f6ebb90400 100644 --- a/activesupport/lib/active_support/core_ext/hash/conversions.rb +++ b/activesupport/lib/active_support/core_ext/hash/conversions.rb @@ -94,6 +94,8 @@ module ActiveSupport #:nodoc: value.to_query(namespace ? "#{namespace}[#{key}]" : key) end.sort * '&' end + + alias_method :to_param, :to_query def to_xml(options = {}) options[:indent] ||= 2 diff --git a/activesupport/lib/active_support/core_ext/time/conversions.rb b/activesupport/lib/active_support/core_ext/time/conversions.rb index 0ce90669d2..ab076a5930 100644 --- a/activesupport/lib/active_support/core_ext/time/conversions.rb +++ b/activesupport/lib/active_support/core_ext/time/conversions.rb @@ -5,6 +5,7 @@ module ActiveSupport #:nodoc: module Conversions DATE_FORMATS = { :db => "%Y-%m-%d %H:%M:%S", + :number => "%Y%m%d%H%M%S", :time => "%H:%M", :short => "%d %b %H:%M", :long => "%B %d, %Y %H:%M", diff --git a/activesupport/lib/active_support/gzip.rb b/activesupport/lib/active_support/gzip.rb new file mode 100644 index 0000000000..c65944dacd --- /dev/null +++ b/activesupport/lib/active_support/gzip.rb @@ -0,0 +1,22 @@ +require 'zlib' +require 'stringio' + +module ActiveSupport + module Gzip + class Stream < StringIO + def close; rewind; end + end + + def self.decompress(source) + Zlib::GzipReader.new(StringIO.new(source)).read + end + + def self.compress(source) + output = Stream.new + gz = Zlib::GzipWriter.new(output) + gz.write(source) + gz.close + output.string + end + end +end
\ No newline at end of file diff --git a/activesupport/lib/active_support/vendor.rb b/activesupport/lib/active_support/vendor.rb index 75c18062c0..6cc7ad8aa1 100644 --- a/activesupport/lib/active_support/vendor.rb +++ b/activesupport/lib/active_support/vendor.rb @@ -12,3 +12,9 @@ begin rescue Gem::LoadError $:.unshift "#{File.dirname(__FILE__)}/vendor/xml-simple-1.0.11" end + +begin + gem 'memcache-client', '~> 1.5.0' +rescue Gem::LoadError + $:.unshift "#{File.dirname(__FILE__)}/vendor/memcache-client-1.5.0" +end
\ No newline at end of file diff --git a/activesupport/lib/active_support/vendor/memcache-client-1.5.0/memcache.rb b/activesupport/lib/active_support/vendor/memcache-client-1.5.0/memcache.rb new file mode 100644 index 0000000000..8c01b2e89d --- /dev/null +++ b/activesupport/lib/active_support/vendor/memcache-client-1.5.0/memcache.rb @@ -0,0 +1,832 @@ +# All original code copyright 2005, 2006, 2007 Bob Cottrell, Eric Hodel, +# The Robot Co-op. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the names of the authors nor the names of their contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS +# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, +# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +require 'socket' +require 'thread' +require 'timeout' +require 'rubygems' + +class String + + ## + # Uses the ITU-T polynomial in the CRC32 algorithm. + + def crc32_ITU_T + n = length + r = 0xFFFFFFFF + + n.times do |i| + r ^= self[i] + 8.times do + if (r & 1) != 0 then + r = (r>>1) ^ 0xEDB88320 + else + r >>= 1 + end + end + end + + r ^ 0xFFFFFFFF + end + +end + +## +# A Ruby client library for memcached. +# +# This is intended to provide access to basic memcached functionality. It +# does not attempt to be complete implementation of the entire API, but it is +# approaching a complete implementation. + +class MemCache + + ## + # The version of MemCache you are using. + + VERSION = '1.5.0' + + ## + # Default options for the cache object. + + DEFAULT_OPTIONS = { + :namespace => nil, + :readonly => false, + :multithread => false, + } + + ## + # Default memcached port. + + DEFAULT_PORT = 11211 + + ## + # Default memcached server weight. + + DEFAULT_WEIGHT = 1 + + ## + # The amount of time to wait for a response from a memcached server. If a + # response is not completed within this time, the connection to the server + # will be closed and an error will be raised. + + attr_accessor :request_timeout + + ## + # The namespace for this instance + + attr_reader :namespace + + ## + # The multithread setting for this instance + + attr_reader :multithread + + ## + # The servers this client talks to. Play at your own peril. + + attr_reader :servers + + ## + # Accepts a list of +servers+ and a list of +opts+. +servers+ may be + # omitted. See +servers=+ for acceptable server list arguments. + # + # Valid options for +opts+ are: + # + # [:namespace] Prepends this value to all keys added or retrieved. + # [:readonly] Raises an exeception on cache writes when true. + # [:multithread] Wraps cache access in a Mutex for thread safety. + # + # Other options are ignored. + + def initialize(*args) + servers = [] + opts = {} + + case args.length + when 0 then # NOP + when 1 then + arg = args.shift + case arg + when Hash then opts = arg + when Array then servers = arg + when String then servers = [arg] + else raise ArgumentError, 'first argument must be Array, Hash or String' + end + when 2 then + servers, opts = args + else + raise ArgumentError, "wrong number of arguments (#{args.length} for 2)" + end + + opts = DEFAULT_OPTIONS.merge opts + @namespace = opts[:namespace] + @readonly = opts[:readonly] + @multithread = opts[:multithread] + @mutex = Mutex.new if @multithread + @buckets = [] + self.servers = servers + end + + ## + # Returns a string representation of the cache object. + + def inspect + "<MemCache: %d servers, %d buckets, ns: %p, ro: %p>" % + [@servers.length, @buckets.length, @namespace, @readonly] + end + + ## + # Returns whether there is at least one active server for the object. + + def active? + not @servers.empty? + end + + ## + # Returns whether or not the cache object was created read only. + + def readonly? + @readonly + end + + ## + # Set the servers that the requests will be distributed between. Entries + # can be either strings of the form "hostname:port" or + # "hostname:port:weight" or MemCache::Server objects. + + def servers=(servers) + # Create the server objects. + @servers = servers.collect do |server| + case server + when String + host, port, weight = server.split ':', 3 + port ||= DEFAULT_PORT + weight ||= DEFAULT_WEIGHT + Server.new self, host, port, weight + when Server + if server.memcache.multithread != @multithread then + raise ArgumentError, "can't mix threaded and non-threaded servers" + end + server + else + raise TypeError, "cannot convert #{server.class} into MemCache::Server" + end + end + + # Create an array of server buckets for weight selection of servers. + @buckets = [] + @servers.each do |server| + server.weight.times { @buckets.push(server) } + end + end + + ## + # Deceremets the value for +key+ by +amount+ and returns the new value. + # +key+ must already exist. If +key+ is not an integer, it is assumed to be + # 0. +key+ can not be decremented below 0. + + def decr(key, amount = 1) + server, cache_key = request_setup key + + if @multithread then + threadsafe_cache_decr server, cache_key, amount + else + cache_decr server, cache_key, amount + end + rescue TypeError, SocketError, SystemCallError, IOError => err + handle_error server, err + end + + ## + # Retrieves +key+ from memcache. If +raw+ is false, the value will be + # unmarshalled. + + def get(key, raw = false) + server, cache_key = request_setup key + + value = if @multithread then + threadsafe_cache_get server, cache_key + else + cache_get server, cache_key + end + + return nil if value.nil? + + value = Marshal.load value unless raw + + return value + rescue TypeError, SocketError, SystemCallError, IOError => err + handle_error server, err + end + + ## + # Retrieves multiple values from memcached in parallel, if possible. + # + # The memcached protocol supports the ability to retrieve multiple + # keys in a single request. Pass in an array of keys to this method + # and it will: + # + # 1. map the key to the appropriate memcached server + # 2. send a single request to each server that has one or more key values + # + # Returns a hash of values. + # + # cache["a"] = 1 + # cache["b"] = 2 + # cache.get_multi "a", "b" # => { "a" => 1, "b" => 2 } + + def get_multi(*keys) + raise MemCacheError, 'No active servers' unless active? + + keys.flatten! + key_count = keys.length + cache_keys = {} + server_keys = Hash.new { |h,k| h[k] = [] } + + # map keys to servers + keys.each do |key| + server, cache_key = request_setup key + cache_keys[cache_key] = key + server_keys[server] << cache_key + end + + results = {} + + server_keys.each do |server, keys| + keys = keys.join ' ' + values = if @multithread then + threadsafe_cache_get_multi server, keys + else + cache_get_multi server, keys + end + values.each do |key, value| + results[cache_keys[key]] = Marshal.load value + end + end + + return results + rescue TypeError, SocketError, SystemCallError, IOError => err + handle_error server, err + end + + ## + # Increments the value for +key+ by +amount+ and retruns the new value. + # +key+ must already exist. If +key+ is not an integer, it is assumed to be + # 0. + + def incr(key, amount = 1) + server, cache_key = request_setup key + + if @multithread then + threadsafe_cache_incr server, cache_key, amount + else + cache_incr server, cache_key, amount + end + rescue TypeError, SocketError, SystemCallError, IOError => err + handle_error server, err + end + + ## + # Add +key+ to the cache with value +value+ that expires in +expiry+ + # seconds. If +raw+ is true, +value+ will not be Marshalled. + # + # Warning: Readers should not call this method in the event of a cache miss; + # see MemCache#add. + + def set(key, value, expiry = 0, raw = false) + raise MemCacheError, "Update of readonly cache" if @readonly + server, cache_key = request_setup key + socket = server.socket + + value = Marshal.dump value unless raw + command = "set #{cache_key} 0 #{expiry} #{value.size}\r\n#{value}\r\n" + + begin + @mutex.lock if @multithread + socket.write command + result = socket.gets + raise MemCacheError, $1.strip if result =~ /^SERVER_ERROR (.*)/ + rescue SocketError, SystemCallError, IOError => err + server.close + raise MemCacheError, err.message + ensure + @mutex.unlock if @multithread + end + end + + ## + # Add +key+ to the cache with value +value+ that expires in +expiry+ + # seconds, but only if +key+ does not already exist in the cache. + # If +raw+ is true, +value+ will not be Marshalled. + # + # Readers should call this method in the event of a cache miss, not + # MemCache#set or MemCache#[]=. + + def add(key, value, expiry = 0, raw = false) + raise MemCacheError, "Update of readonly cache" if @readonly + server, cache_key = request_setup key + socket = server.socket + + value = Marshal.dump value unless raw + command = "add #{cache_key} 0 #{expiry} #{value.size}\r\n#{value}\r\n" + + begin + @mutex.lock if @multithread + socket.write command + socket.gets + rescue SocketError, SystemCallError, IOError => err + server.close + raise MemCacheError, err.message + ensure + @mutex.unlock if @multithread + end + end + + ## + # Removes +key+ from the cache in +expiry+ seconds. + + def delete(key, expiry = 0) + @mutex.lock if @multithread + + raise MemCacheError, "No active servers" unless active? + cache_key = make_cache_key key + server = get_server_for_key cache_key + + sock = server.socket + raise MemCacheError, "No connection to server" if sock.nil? + + begin + sock.write "delete #{cache_key} #{expiry}\r\n" + sock.gets + rescue SocketError, SystemCallError, IOError => err + server.close + raise MemCacheError, err.message + end + ensure + @mutex.unlock if @multithread + end + + ## + # Flush the cache from all memcache servers. + + def flush_all + raise MemCacheError, 'No active servers' unless active? + raise MemCacheError, "Update of readonly cache" if @readonly + begin + @mutex.lock if @multithread + @servers.each do |server| + begin + sock = server.socket + raise MemCacheError, "No connection to server" if sock.nil? + sock.write "flush_all\r\n" + result = sock.gets + raise MemCacheError, $2.strip if result =~ /^(SERVER_)?ERROR(.*)/ + rescue SocketError, SystemCallError, IOError => err + server.close + raise MemCacheError, err.message + end + end + ensure + @mutex.unlock if @multithread + end + end + + ## + # Reset the connection to all memcache servers. This should be called if + # there is a problem with a cache lookup that might have left the connection + # in a corrupted state. + + def reset + @servers.each { |server| server.close } + end + + ## + # Returns statistics for each memcached server. An explanation of the + # statistics can be found in the memcached docs: + # + # http://code.sixapart.com/svn/memcached/trunk/server/doc/protocol.txt + # + # Example: + # + # >> pp CACHE.stats + # {"localhost:11211"=> + # {"bytes"=>4718, + # "pid"=>20188, + # "connection_structures"=>4, + # "time"=>1162278121, + # "pointer_size"=>32, + # "limit_maxbytes"=>67108864, + # "cmd_get"=>14532, + # "version"=>"1.2.0", + # "bytes_written"=>432583, + # "cmd_set"=>32, + # "get_misses"=>0, + # "total_connections"=>19, + # "curr_connections"=>3, + # "curr_items"=>4, + # "uptime"=>1557, + # "get_hits"=>14532, + # "total_items"=>32, + # "rusage_system"=>0.313952, + # "rusage_user"=>0.119981, + # "bytes_read"=>190619}} + # => nil + + def stats + raise MemCacheError, "No active servers" unless active? + server_stats = {} + + @servers.each do |server| + sock = server.socket + raise MemCacheError, "No connection to server" if sock.nil? + + value = nil + begin + sock.write "stats\r\n" + stats = {} + while line = sock.gets do + break if line == "END\r\n" + if line =~ /^STAT ([\w]+) ([\w\.\:]+)/ then + name, value = $1, $2 + stats[name] = case name + when 'version' + value + when 'rusage_user', 'rusage_system' then + seconds, microseconds = value.split(/:/, 2) + microseconds ||= 0 + Float(seconds) + (Float(microseconds) / 1_000_000) + else + if value =~ /^\d+$/ then + value.to_i + else + value + end + end + end + end + server_stats["#{server.host}:#{server.port}"] = stats + rescue SocketError, SystemCallError, IOError => err + server.close + raise MemCacheError, err.message + end + end + + server_stats + end + + ## + # Shortcut to get a value from the cache. + + alias [] get + + ## + # Shortcut to save a value in the cache. This method does not set an + # expiration on the entry. Use set to specify an explicit expiry. + + def []=(key, value) + set key, value + end + + protected + + ## + # Create a key for the cache, incorporating the namespace qualifier if + # requested. + + def make_cache_key(key) + if namespace.nil? then + key + else + "#{@namespace}:#{key}" + end + end + + ## + # Pick a server to handle the request based on a hash of the key. + + def get_server_for_key(key) + raise ArgumentError, "illegal character in key #{key.inspect}" if + key =~ /\s/ + raise ArgumentError, "key too long #{key.inspect}" if key.length > 250 + raise MemCacheError, "No servers available" if @servers.empty? + return @servers.first if @servers.length == 1 + + hkey = hash_for key + + 20.times do |try| + server = @buckets[hkey % @buckets.nitems] + return server if server.alive? + hkey += hash_for "#{try}#{key}" + end + + raise MemCacheError, "No servers available" + end + + ## + # Returns an interoperable hash value for +key+. (I think, docs are + # sketchy for down servers). + + def hash_for(key) + (key.crc32_ITU_T >> 16) & 0x7fff + end + + ## + # Performs a raw decr for +cache_key+ from +server+. Returns nil if not + # found. + + def cache_decr(server, cache_key, amount) + socket = server.socket + socket.write "decr #{cache_key} #{amount}\r\n" + text = socket.gets + return nil if text == "NOT_FOUND\r\n" + return text.to_i + end + + ## + # Fetches the raw data for +cache_key+ from +server+. Returns nil on cache + # miss. + + def cache_get(server, cache_key) + socket = server.socket + socket.write "get #{cache_key}\r\n" + keyline = socket.gets # "VALUE <key> <flags> <bytes>\r\n" + + if keyline.nil? then + server.close + raise MemCacheError, "lost connection to #{server.host}:#{server.port}" + end + + return nil if keyline == "END\r\n" + + unless keyline =~ /(\d+)\r/ then + server.close + raise MemCacheError, "unexpected response #{keyline.inspect}" + end + value = socket.read $1.to_i + socket.read 2 # "\r\n" + socket.gets # "END\r\n" + return value + end + + ## + # Fetches +cache_keys+ from +server+ using a multi-get. + + def cache_get_multi(server, cache_keys) + values = {} + socket = server.socket + socket.write "get #{cache_keys}\r\n" + + while keyline = socket.gets do + return values if keyline == "END\r\n" + + unless keyline =~ /^VALUE (.+) (.+) (.+)/ then + server.close + raise MemCacheError, "unexpected response #{keyline.inspect}" + end + + key, data_length = $1, $3 + values[$1] = socket.read data_length.to_i + socket.read(2) # "\r\n" + end + + server.close + raise MemCacheError, "lost connection to #{server.host}:#{server.port}" + end + + ## + # Performs a raw incr for +cache_key+ from +server+. Returns nil if not + # found. + + def cache_incr(server, cache_key, amount) + socket = server.socket + socket.write "incr #{cache_key} #{amount}\r\n" + text = socket.gets + return nil if text == "NOT_FOUND\r\n" + return text.to_i + end + + ## + # Handles +error+ from +server+. + + def handle_error(server, error) + server.close if server + new_error = MemCacheError.new error.message + new_error.set_backtrace error.backtrace + raise new_error + end + + ## + # Performs setup for making a request with +key+ from memcached. Returns + # the server to fetch the key from and the complete key to use. + + def request_setup(key) + raise MemCacheError, 'No active servers' unless active? + cache_key = make_cache_key key + server = get_server_for_key cache_key + raise MemCacheError, 'No connection to server' if server.socket.nil? + return server, cache_key + end + + def threadsafe_cache_decr(server, cache_key, amount) # :nodoc: + @mutex.lock + cache_decr server, cache_key, amount + ensure + @mutex.unlock + end + + def threadsafe_cache_get(server, cache_key) # :nodoc: + @mutex.lock + cache_get server, cache_key + ensure + @mutex.unlock + end + + def threadsafe_cache_get_multi(socket, cache_keys) # :nodoc: + @mutex.lock + cache_get_multi socket, cache_keys + ensure + @mutex.unlock + end + + def threadsafe_cache_incr(server, cache_key, amount) # :nodoc: + @mutex.lock + cache_incr server, cache_key, amount + ensure + @mutex.unlock + end + + ## + # This class represents a memcached server instance. + + class Server + + ## + # The amount of time to wait to establish a connection with a memcached + # server. If a connection cannot be established within this time limit, + # the server will be marked as down. + + CONNECT_TIMEOUT = 0.25 + + ## + # The amount of time to wait before attempting to re-establish a + # connection with a server that is marked dead. + + RETRY_DELAY = 30.0 + + ## + # The host the memcached server is running on. + + attr_reader :host + + ## + # The port the memcached server is listening on. + + attr_reader :port + + ## + # The weight given to the server. + + attr_reader :weight + + ## + # The time of next retry if the connection is dead. + + attr_reader :retry + + ## + # A text status string describing the state of the server. + + attr_reader :status + + ## + # Create a new MemCache::Server object for the memcached instance + # listening on the given host and port, weighted by the given weight. + + def initialize(memcache, host, port = DEFAULT_PORT, weight = DEFAULT_WEIGHT) + raise ArgumentError, "No host specified" if host.nil? or host.empty? + raise ArgumentError, "No port specified" if port.nil? or port.to_i.zero? + + @memcache = memcache + @host = host + @port = port.to_i + @weight = weight.to_i + + @multithread = @memcache.multithread + @mutex = Mutex.new + + @sock = nil + @retry = nil + @status = 'NOT CONNECTED' + end + + ## + # Return a string representation of the server object. + + def inspect + "<MemCache::Server: %s:%d [%d] (%s)>" % [@host, @port, @weight, @status] + end + + ## + # Check whether the server connection is alive. This will cause the + # socket to attempt to connect if it isn't already connected and or if + # the server was previously marked as down and the retry time has + # been exceeded. + + def alive? + !!socket + end + + ## + # Try to connect to the memcached server targeted by this object. + # Returns the connected socket object on success or nil on failure. + + def socket + @mutex.lock if @multithread + return @sock if @sock and not @sock.closed? + + @sock = nil + + # If the host was dead, don't retry for a while. + return if @retry and @retry > Time.now + + # Attempt to connect if not already connected. + begin + @sock = timeout CONNECT_TIMEOUT do + TCPSocket.new @host, @port + end + if Socket.constants.include? 'TCP_NODELAY' then + @sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1 + end + @retry = nil + @status = 'CONNECTED' + rescue SocketError, SystemCallError, IOError, Timeout::Error => err + mark_dead err.message + end + + return @sock + ensure + @mutex.unlock if @multithread + end + + ## + # Close the connection to the memcached server targeted by this + # object. The server is not considered dead. + + def close + @mutex.lock if @multithread + @sock.close if @sock && !@sock.closed? + @sock = nil + @retry = nil + @status = "NOT CONNECTED" + ensure + @mutex.unlock if @multithread + end + + private + + ## + # Mark the server as dead and close its socket. + + def mark_dead(reason = "Unknown error") + @sock.close if @sock && !@sock.closed? + @sock = nil + @retry = Time.now + RETRY_DELAY + + @status = sprintf "DEAD: %s, will retry at %s", reason, @retry + end + + end + + ## + # Base MemCache exception class. + + class MemCacheError < RuntimeError; end + +end + diff --git a/activesupport/test/caching_test.rb b/activesupport/test/caching_test.rb new file mode 100644 index 0000000000..592eede63e --- /dev/null +++ b/activesupport/test/caching_test.rb @@ -0,0 +1,27 @@ +require File.dirname(__FILE__) + '/abstract_unit' + +class CacheStoreSettingTest < Test::Unit::TestCase + def test_file_fragment_cache_store + store = ActiveSupport::Cache.lookup_store :file_store, "/path/to/cache/directory" + assert_kind_of(ActiveSupport::Cache::FileStore, store) + assert_equal "/path/to/cache/directory", store.cache_path + end + + def test_drb_fragment_cache_store + store = ActiveSupport::Cache.lookup_store :drb_store, "druby://localhost:9192" + assert_kind_of(ActiveSupport::Cache::DRbStore, store) + assert_equal "druby://localhost:9192", store.address + end + + def test_mem_cache_fragment_cache_store + store = ActiveSupport::Cache.lookup_store :mem_cache_store, "localhost" + assert_kind_of(ActiveSupport::Cache::MemCacheStore, store) + assert_equal %w(localhost), store.addresses + end + + def test_object_assigned_fragment_cache_store + store = ActiveSupport::Cache.lookup_store ActiveSupport::Cache::FileStore.new("/path/to/cache/directory") + assert_kind_of(ActiveSupport::Cache::FileStore, store) + assert_equal "/path/to/cache/directory", store.cache_path + end +end |