diff --git a/Gemfile b/Gemfile index c7bb0be6ea50732c909869192656110df389aedc..d08a2d3a1855f80b51033a36df014e95cce49c78 100644 --- a/Gemfile +++ b/Gemfile @@ -77,7 +77,12 @@ gem 'SystemTimer', '1.2.1', :platforms => :ruby_18 gem 'hoptoad_notifier' gem 'newrelic_rpm', :require => false +# statistics + +gem 'statsample', :require => false + #mail + gem 'messagebus_ruby_api', '0.4.8' # tags diff --git a/Gemfile.lock b/Gemfile.lock index 68bab41628c34f798e70f2d6175f6229f316f42f..8a33b662c7941e11779f6b56faf20151fff8656f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -139,6 +139,7 @@ GEM uuidtools childprocess (0.2.2) ffi (~> 1.0.6) + clbustos-rtf (0.4.2) closure-compiler (1.1.4) cloudfiles (1.4.10) mime-types (>= 1.16) @@ -163,12 +164,15 @@ GEM devise (~> 1.3.1) rails (<= 3.2, >= 3.0.0) diff-lcs (1.1.3) + dirty-memoize (0.0.4) + distribution (0.6.0) em-synchrony (0.2.0) eventmachine (>= 0.12.9) erubis (2.6.6) abstract (>= 1.0.0) eventmachine (0.12.10) excon (0.2.4) + extendmatrix (0.3.1) extlib (0.9.15) factory_girl (2.1.2) activesupport @@ -249,6 +253,8 @@ GEM mime-types (1.16) mini_magick (3.2) subexec (~> 0.0.4) + minimization (0.2.1) + text-table (~> 1.2) mixlib-authentication (1.1.4) mixlib-log mixlib-cli (1.2.2) @@ -331,6 +337,16 @@ GEM parallel_tests (0.6.7) parallel polyglot (0.3.2) + prawn (0.8.4) + prawn-core (< 0.9, >= 0.8.4) + prawn-layout (< 0.9, >= 0.8.4) + prawn-security (< 0.9, >= 0.8.4) + prawn-core (0.8.4) + prawn-layout (0.8.4) + prawn-security (0.8.4) + prawn-svg (0.9.1.10) + prawn (>= 0.8.4) + prawn-core (>= 0.8.4) pyu-ruby-sasl (0.0.3.3) rack (1.2.4) rack-mobile-detect (0.3.0) @@ -366,6 +382,11 @@ GEM redis (2.2.2) redis-namespace (0.8.0) redis (< 3.0.0) + reportbuilder (1.4.1) + clbustos-rtf (~> 0.4.0) + prawn (~> 0.8.4) + prawn-svg (~> 0.9.1) + text-table (~> 1.2) resque (1.10.0) json (~> 1.4.6) redis-namespace (~> 0.8.0) @@ -378,6 +399,7 @@ GEM resque (~> 1.0) rest-client (1.6.1) mime-types (>= 1.16) + rserve-client (0.2.5) rspec (2.6.0) rspec-core (~> 2.6.0) rspec-expectations (~> 2.6.0) @@ -406,6 +428,7 @@ GEM linecache19 (>= 0.5.11) ruby-debug-base19 (>= 0.11.19) ruby-hmac (0.4.0) + ruby-ole (1.2.11.2) ruby-openid (2.1.8) ruby-openid-apps-discovery (1.2.0) ruby-openid (>= 2.1.7) @@ -413,6 +436,7 @@ GEM ruby_core_source (0.1.5) archive-tar-minitar (>= 0.5.2) rubyntlm (0.1.1) + rubyvis (0.4.1) rubyzip (0.9.4) sass (3.1.7) selenium-webdriver (2.7.0) @@ -425,10 +449,26 @@ GEM sinatra (1.2.7) rack (~> 1.1) tilt (>= 1.2.2, < 2.0) + spreadsheet (0.6.5.9) + ruby-ole (>= 1.0) sqlite3 (1.3.4) + statsample (1.1.0) + dirty-memoize (~> 0.0) + distribution (~> 0.3) + extendmatrix (~> 0.3.1) + fastercsv (> 0) + minimization (~> 0.2.0) + reportbuilder (~> 1.4) + rserve-client (~> 0.2.5) + rubyvis (~> 0.4.0) + spreadsheet (~> 0.6.5) + statsample-bivariate-extension (> 0) + statsample-bivariate-extension (1.1.0) + distribution (~> 0.6) subexec (0.0.4) systemu (2.4.0) term-ansicolor (1.0.6) + text-table (1.2.2) thin (1.2.11) daemons (>= 1.0.9) eventmachine (>= 0.12.6) @@ -537,6 +577,7 @@ DEPENDENCIES settingslogic (= 2.0.6) sod! sqlite3 + statsample thin (= 1.2.11) twitter (= 1.5.0) typhoeus diff --git a/app/controllers/admins_controller.rb b/app/controllers/admins_controller.rb index ad11611736f5b0d8fa0a99c5d987012777a1da11..9fe6d24beeeb78aa46bddc5673a49766d0a7b41f 100644 --- a/app/controllers/admins_controller.rb +++ b/app/controllers/admins_controller.rb @@ -63,6 +63,21 @@ class AdminsController < ApplicationController #@posts[:new_public] = Post.where(:type => ['StatusMessage','ActivityStreams::Photo'], # :public => true).order('created_at DESC').limit(15).all + end + + def correlations + + + @correlations = Statistics.generate_correlations(params[:number_of_weeks]) + + 5.times.inject({}) do |stats, n| + week_start = (Time.now - n.weeks).beginning_of_week + week_end = week_start - 1.week + stats[week_start] = Statistics.new(week_start, week_end).generate_correlations + end + + + end private diff --git a/lib/statistics.rb b/lib/statistics.rb new file mode 100644 index 0000000000000000000000000000000000000000..00c80086f6b39ad6907cb12f33c6cd41f23777a7 --- /dev/null +++ b/lib/statistics.rb @@ -0,0 +1,96 @@ +require 'statsample' + +class Statistics + + attr_reader :start_time, + :range + + def initialize + #@start_time = start_time + #@range = range + end + + def posts_count_sql + <<SQL + SELECT users.id AS id, count(posts.id) AS count + FROM users + JOIN people ON people.owner_id = users.id + LEFT OUTER JOIN posts ON people.id = posts.author_id + GROUP BY users.id +SQL + end + + def invites_sent_count_sql + <<SQL + SELECT users.id AS id, count(invitations.id) AS count + FROM users + LEFT OUTER JOIN invitations ON users.id = invitations.sender_id + GROUP BY users.id +SQL + end + + def tags_followed_count_sql + <<SQL + SELECT users.id AS id, count(tag_followings.id) AS count + FROM users + LEFT OUTER JOIN tag_followings on users.id = tag_followings.user_id + GROUP BY users.id +SQL + end + + def mentions_count_sql + <<SQL + SELECT users.id AS id, count(mentions.id) AS count + FROM users + JOIN people on users.id = people.owner_id + LEFT OUTER JOIN mentions on people.id = mentions.person_id + GROUP BY users.id +SQL + end + + def sign_in_count_sql + <<SQL + SELECT users.id AS id, users.sign_in_count AS count + FROM users +SQL + end + + def posts_count_correlation + + # [{"id" => 1 , "count" => 123}] + post_count_array = User.connection.select_all(self.posts_count_sql) + + post_count_hash = {} + post_count_array.each{ |h| post_count_hash[h[id]] = h["count"]} + + + end + + + ###\ + #def correlate(thing) + # sql = self.send("#{thing}_count_sql".to_sym) + # self.correlation(User.connection.select_all(sql), + #end + + + ### + + def correlation(x_array, y_array) + x = x_array.to_scale + y = y_array.to_scale + pearson = Statsample::Bivariate::Pearson.new(x,y) + pearson.r + end + + ### % of cohort came back last week + def retention(n) + week_created(n).where("current_sign_in_at > ?", Time.now - 1.week).count.to_f/week_created(n).count + end + + protected + def week_created(n) + User.where("username IS NOT NULL").where("created_at > ? and created_at < ?", Time.now - (n+1).weeks, Time.now - n.weeks) + end + +end diff --git a/spec/lib/statistics_spec.rb b/spec/lib/statistics_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..77a62bf9fdb7316605ee703554400bc293003ddf --- /dev/null +++ b/spec/lib/statistics_spec.rb @@ -0,0 +1,96 @@ +require 'spec_helper' +require 'lib/statistics' + +describe Statistics do + + before do + @stats = Statistics.new(time, time - 1.week) + @result = [{"id" => alice.id , "count" => 0 }, + {"id" => bob.id , "count" => 1 }, + {"id" => eve.id , "count" => 0 }, + {"id" => local_luke.id , "count" => 0 }, + {"id" => local_leia.id , "count" => 0 }, + ] + end + + describe '#posts_count_sql' do + it "pulls back an array of post counts and ids" do + Factory.create(:status_message, :author => bob.person) + User.connection.select_all(@stats.posts_count_sql).should =~ @result + end + end + + describe '#invites_sent_count_sql' do + it "pulls back an array of invite counts and ids" do + Invitation.batch_invite(["a@a.com"], :sender => bob, :aspect => bob.aspects.first, :service => 'email') + User.connection.select_all(@stats.invites_sent_count_sql).should =~ @result + end + end + + describe '#tags_followed_count_sql' do + it "pulls back an array of tag following counts and ids" do + TagFollowing.create!(:user => bob, :tag_id => 1) + User.connection.select_all(@stats.tags_followed_count_sql).should =~ @result + end + end + + describe '#mentions_count_sql' do + it "pulls back an array of mentions following counts and ids" do + post = Factory.create(:status_message, :author => bob.person) + Mention.create(:post => post, :person => bob.person) + User.connection.select_all(@stats.mentions_count_sql).should =~ @result + end + end + + describe '#sign_in_count_sql' do + it "pulls back an array of sign_in_counts and ids" do + bob.sign_in_count = 1 + bob.save! + User.connection.select_all(@stats.sign_in_count_sql).should =~ @result + end + end + + describe "#correlation" do + it 'returns the correlation coefficient' do + @stats.correlation([1,2],[1,2]).to_s.should == 1.0.to_s + @stats.correlation([1,2,1,2],[1,1,2,2]).to_s.should == 0.0.to_s + end + end + describe "#correlation_hash" do + + it 'it returns a hash of including start and end time' do + time = Time.now + + hash = @stats.correlation_hash + hash[:starrt_time].should == time + hash[:end_time].should == time - 1.week + end + + it 'returns the post count (and sign_in_count) correlation' do + @stats.stub(:posts_count_correlation).and_return(0.5) + + @stats.generate_correlations[:posts_count].should == 0.5 + end + end + + context 'todos' do + before do + pending + end + + # requires a threshold + + describe '#disabled_email_count_sql' do + end + + # binary things + describe '#completed_getting_started_count_sql' do + end + + describe 'used_cubbies_sql' do + end + + describe '.sign_up_method_sql' do + end + end +end