Skip to content

Commit 8885300

Browse files
committed
add group_by method
1 parent ca540b2 commit 8885300

22 files changed

+287
-5
lines changed

.rspec

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
--color
2+
--tty
3+
--format progress
4+
--order random
5+
--backtrace

Appraisals

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
appraise 'rails3.1' do
2+
gem 'activesupport', '~> 3.1.0'
3+
gem 'activerecord', '~> 3.1.0'
4+
end
5+
6+
appraise 'rails3.2' do
7+
gem 'activesupport', '~> 3.2.0'
8+
gem 'activerecord', '~> 3.2.0'
9+
end
10+
11+
appraise 'rails4.0' do
12+
gem 'activesupport', '~> 4.0.0'
13+
gem 'activerecord', '~> 4.0.0'
14+
end
15+
16+
appraise 'rails4.1' do
17+
gem 'activesupport', '~> 4.1.0'
18+
gem 'activerecord', '~> 4.1.0'
19+
end

Gemfile

+5
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
source 'https://rubygems.org'
22

33
# Specify your gem's dependencies in redis_counters-dumpers.gemspec
4+
5+
group :development, :test do
6+
gem 'combustion', github: 'pat/combustion', ref: '7d0d24c3f36ce0eb336177fc493be0721bc26665'
7+
end
8+
49
gemspec

Makefile

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
RAILS_ENV = test
2+
BUNDLE = RAILS_ENV=${RAILS_ENV} bundle
3+
BUNDLE_OPTIONS = -j 2
4+
RSPEC = rspec
5+
APPRAISAL = appraisal
6+
7+
all: test
8+
9+
test: config/database bundler/install appraisal/install
10+
${BUNDLE} exec ${APPRAISAL} ${RSPEC} spec 2>&1
11+
12+
config/database:
13+
touch spec/internal/config/database.yml
14+
echo 'test:' >> spec/internal/config/database.yml
15+
echo ' adapter: postgresql' >> spec/internal/config/database.yml
16+
echo ' database: docker' >> spec/internal/config/database.yml
17+
echo ' username: docker' >> spec/internal/config/database.yml
18+
echo ' host: localhost' >> spec/internal/config/database.yml
19+
echo ' min_messages: warning' >> spec/internal/config/database.yml
20+
21+
bundler/install:
22+
if ! gem list bundler -i > /dev/null; then \
23+
gem install bundler; \
24+
fi
25+
${BUNDLE} install ${BUNDLE_OPTIONS}
26+
27+
appraisal/install:
28+
${BUNDLE} exec ${APPRAISAL} install

gemfiles/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
*.gemfile.lock

gemfiles/rails3.1.gemfile

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# This file was generated by Appraisal
2+
3+
source "https://rubygems.org"
4+
5+
gem "activesupport", "~> 3.1.0"
6+
gem "activerecord", "~> 3.1.0"
7+
8+
group :development, :test do
9+
gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665"
10+
end
11+
12+
gemspec :path => "../"

gemfiles/rails3.2.gemfile

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# This file was generated by Appraisal
2+
3+
source "https://rubygems.org"
4+
5+
gem "activesupport", "~> 3.2.0"
6+
gem "activerecord", "~> 3.2.0"
7+
8+
group :development, :test do
9+
gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665"
10+
end
11+
12+
gemspec :path => "../"

gemfiles/rails4.0.gemfile

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# This file was generated by Appraisal
2+
3+
source "https://rubygems.org"
4+
5+
gem "activesupport", "~> 4.0.0"
6+
gem "activerecord", "~> 4.0.0"
7+
8+
group :development, :test do
9+
gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665"
10+
end
11+
12+
gemspec :path => "../"

gemfiles/rails4.1.gemfile

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# This file was generated by Appraisal
2+
3+
source "https://rubygems.org"
4+
5+
gem "activesupport", "~> 4.1.0"
6+
gem "activerecord", "~> 4.1.0"
7+
8+
group :development, :test do
9+
gem "combustion", :github => "pat/combustion", :ref => "7d0d24c3f36ce0eb336177fc493be0721bc26665"
10+
end
11+
12+
gemspec :path => "../"

lib/redis_counters/dumpers.rb

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
require 'active_record'
12
require 'redis_counters/dumpers/version'
23

34
module RedisCounters

lib/redis_counters/dumpers/destination.rb

+15-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# coding: utf-8
22
require 'forwardable'
33
require 'active_support/core_ext/hash/indifferent_access'
4+
require 'active_support/core_ext/object/blank'
45
require_relative 'dsl/destination'
56

67
module RedisCounters
@@ -48,6 +49,9 @@ class Destination
4849
# а целевое поле :date, указывает на поле :start_month_date, дампера.
4950
attr_accessor :fields_map
5051

52+
# Список полей по которым будет группироваться таблицы с исходными данным, Array
53+
attr_accessor :group_by
54+
5155
# Список дополнительных условий, которые применяются при обновлении целевой таблицы, Array of String.
5256
# Каждое условие представляет собой строку - часть SQL выражения, которое может включать именованные
5357
# параметры из числа доступных в хеше оббщих параметров дампера: engine.common_params.
@@ -68,7 +72,8 @@ def merge
6872
source AS
6973
(
7074
SELECT #{selected_fields_expression}
71-
FROM #{source_table}
75+
FROM #{source_table}
76+
#{group_by_expression}
7277
),
7378
updated AS
7479
(
@@ -83,11 +88,11 @@ def merge
8388
INSERT INTO #{target_table} (#{target_fields})
8489
SELECT #{target_fields}
8590
FROM source
86-
WHERE NOT EXISTS (
87-
SELECT 1
91+
WHERE NOT EXISTS (
92+
SELECT 1
8893
FROM updated target
89-
WHERE #{matching_expression}
90-
#{extra_conditions}
94+
WHERE #{matching_expression}
95+
#{extra_conditions}
9196
)
9297
SQL
9398

@@ -105,6 +110,11 @@ def selected_fields_expression
105110
full_fields_map.map { |target_field, source_field| "#{source_field} as #{target_field}" }.join(', ')
106111
end
107112

113+
def group_by_expression
114+
return if group_by.blank?
115+
'GROUP BY %s' % [group_by.join(', ')]
116+
end
117+
108118
def full_fields_map
109119
fields_map.reverse_merge(Hash[fields.zip(fields)])
110120
end

lib/redis_counters/dumpers/dsl/destination.rb

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ class Configuration < ::RedisCounters::Dumpers::Dsl::Base
1717
varags_setter :fields
1818
varags_setter :key_fields
1919
varags_setter :increment_fields
20+
varags_setter :group_by
2021

2122
alias_method :take, :fields
2223

lib/redis_counters/dumpers/engine.rb

+7
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
require 'active_support/core_ext/hash/indifferent_access'
55
require 'redis'
66
require 'redis/namespace'
7+
require 'redis_counters'
78
require_relative 'dsl/engine'
89

910
module RedisCounters
@@ -179,6 +180,8 @@ def merge_data
179180
destinations.each { |dest| dest.merge }
180181

181182
fire_callback(:on_after_merge, self, db_connection)
183+
184+
drop_temp_table
182185
end
183186

184187
def fill_temp_table
@@ -245,6 +248,10 @@ def create_temp_table
245248
SQL
246249
end
247250

251+
def drop_temp_table
252+
db_connection.execute "DROP TABLE #{temp_table_name}"
253+
end
254+
248255
def analyze_table
249256
db_connection.execute <<-SQL
250257
ANALYZE #{temp_table_name}

redis_counters-dumpers.gemspec

+9
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,19 @@ Gem::Specification.new do |spec|
1919

2020
spec.add_dependency 'activesupport', '>= 3.0'
2121
spec.add_dependency 'activerecord', '>= 3.0'
22+
spec.add_dependency 'pg'
2223
spec.add_dependency 'redis', '>= 3.0'
2324
spec.add_dependency 'redis-namespace', '>= 1.3'
2425
spec.add_dependency 'callbacks_rb', '>= 0.0.1'
26+
spec.add_dependency 'redis_counters', '>= 1.3'
27+
2528

2629
spec.add_development_dependency 'bundler', '>= 1.7'
2730
spec.add_development_dependency 'rake', '>= 10.0'
31+
spec.add_development_dependency 'rspec', '>= 3.2'
32+
spec.add_development_dependency 'rspec-rails', '>= 3.2'
33+
spec.add_development_dependency 'rspec-given', '>= 3.5'
34+
spec.add_development_dependency 'appraisal', '>= 1.0.2'
35+
spec.add_development_dependency 'mock_redis'
36+
spec.add_development_dependency 'apress-changelogger'
2837
end
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
class StatsAggTotal < ActiveRecord::Base
2+
end
+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
class StatsByDay < ActiveRecord::Base
2+
end
+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
class StatsTotal < ActiveRecord::Base
2+
end

spec/internal/config/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
database.yml

spec/internal/db/schema.rb

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
ActiveRecord::Schema.define do
2+
create_table :stats_by_days do |t|
3+
t.integer :record_id, null: false
4+
t.integer :column_id, null: false
5+
t.date :date, null: false
6+
t.integer :hits, null: false, default: 0
7+
end
8+
9+
add_index :stats_by_days, [:record_id, :column_id, :date], unique: true
10+
11+
create_table :stats_totals do |t|
12+
t.integer :record_id, null: false
13+
t.integer :column_id, null: false
14+
t.integer :hits, null: false, default: 0
15+
end
16+
17+
add_index :stats_totals, [:record_id, :column_id], unique: true
18+
19+
create_table :stats_agg_totals do |t|
20+
t.integer :record_id, null: false
21+
t.integer :hits, null: false, default: 0
22+
end
23+
24+
add_index :stats_agg_totals, [:record_id], unique: true
25+
end

spec/internal/log/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
*.log
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
require 'spec_helper'
2+
3+
describe RedisCounters::Dumpers::Engine do
4+
let(:dumper) do
5+
RedisCounters::Dumpers::Engine.build do
6+
name :stats_totals
7+
fields record_id: :integer,
8+
column_id: :integer,
9+
value: :integer,
10+
date: :date
11+
12+
destination do
13+
model StatsByDay
14+
take :record_id, :column_id, :hits, :date
15+
key_fields :record_id, :column_id, :date
16+
increment_fields :hits
17+
map :hits, to: :value
18+
condition 'target.date = :date'
19+
end
20+
21+
destination do
22+
model StatsTotal
23+
take :record_id, :column_id, :hits
24+
key_fields :record_id, :column_id
25+
increment_fields :hits
26+
map :hits, to: :value
27+
end
28+
29+
destination do
30+
model StatsAggTotal
31+
take :record_id, :hits
32+
key_fields :record_id
33+
increment_fields :hits
34+
map :hits, to: 'sum(value)'
35+
group_by :record_id
36+
end
37+
38+
on_before_merge do |dumper, _connection|
39+
dumper.common_params = {date: dumper.date.strftime('%Y-%m-%d')}
40+
end
41+
end
42+
end
43+
44+
let(:prev_date) { Date.new(2015, 1, 19) }
45+
let(:prev_date_s) { prev_date.strftime('%Y-%m-%d') }
46+
47+
let(:date) { Date.new(2015, 1, 20) }
48+
let(:date_s) { date.strftime('%Y-%m-%d') }
49+
50+
let(:counter) do
51+
RedisCounters.create_counter(Redis.current,
52+
counter_class: RedisCounters::HashCounter,
53+
counter_name: :record_hits_by_day,
54+
group_keys: [:record_id, :column_id],
55+
partition_keys: [:date]
56+
)
57+
end
58+
59+
before do
60+
allow(dumper).to receive(:redis_session).and_return(MockRedis.new)
61+
end
62+
63+
describe '#process!' do
64+
before do
65+
counter.increment(date: prev_date_s, record_id: 1, column_id: 100)
66+
counter.increment(date: prev_date_s, record_id: 1, column_id: 200)
67+
counter.increment(date: prev_date_s, record_id: 1, column_id: 200)
68+
counter.increment(date: prev_date_s, record_id: 2, column_id: 100)
69+
70+
dumper.process!(counter, prev_date)
71+
72+
counter.increment(date: date_s, record_id: 1, column_id: 100)
73+
counter.increment(date: date_s, record_id: 1, column_id: 200)
74+
counter.increment(date: date_s, record_id: 1, column_id: 200)
75+
counter.increment(date: date_s, record_id: 2, column_id: 100)
76+
77+
dumper.process!(counter, date)
78+
end
79+
80+
Then { expect(StatsByDay.count).to eq 6 }
81+
And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: prev_date).first.hits).to eq 1 }
82+
And { expect(StatsByDay.where(record_id: 1, column_id: 200, date: prev_date).first.hits).to eq 2 }
83+
And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: prev_date).first.hits).to eq 1 }
84+
And { expect(StatsByDay.where(record_id: 1, column_id: 100, date: date).first.hits).to eq 1 }
85+
And { expect(StatsByDay.where(record_id: 1, column_id: 200, date: date).first.hits).to eq 2 }
86+
And { expect(StatsByDay.where(record_id: 2, column_id: 100, date: date).first.hits).to eq 1 }
87+
88+
And { expect(StatsTotal.count).to eq 3 }
89+
And { expect(StatsTotal.where(record_id: 1, column_id: 100).first.hits).to eq 2 }
90+
And { expect(StatsTotal.where(record_id: 1, column_id: 200).first.hits).to eq 4 }
91+
And { expect(StatsTotal.where(record_id: 2, column_id: 100).first.hits).to eq 2 }
92+
93+
And { expect(StatsAggTotal.count).to eq 2 }
94+
And { expect(StatsAggTotal.where(record_id: 1).first.hits).to eq 6 }
95+
And { expect(StatsAggTotal.where(record_id: 2).first.hits).to eq 2 }
96+
end
97+
end

spec/spec_helper.rb

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# coding: utf-8
2+
require 'bundler/setup'
3+
require 'redis_counters/dumpers'
4+
5+
require 'combustion'
6+
Combustion.initialize! :active_record
7+
8+
require 'rspec/rails'
9+
require 'rspec/given'
10+
11+
require 'mock_redis'
12+
require 'redis'
13+
Redis.current = MockRedis.new
14+
15+
RSpec.configure do |config|
16+
config.use_transactional_fixtures = true
17+
config.before { Redis.current.flushdb }
18+
end

0 commit comments

Comments
 (0)