Added oEmbed support

Gemfile

@@ -64,6 +64,7 @@ gem 'rails-i18n'
 gem 'nokogiri'
 gem 'redcarpet', "2.0.0b5"
 gem 'roxml', :git => 'git://github.com/Empact/roxml.git', :ref => '7ea9a9ffd2338aaef5b0'
+gem 'ruby-oembed'
 
 # queue
 

app/helpers/markdownify_helper.rb

@@ -33,7 +33,11 @@ module MarkdownifyHelper
     return '' if message.blank?
 
     #renderer = Redcarpet::Render::HTML.new(render_options)
-    renderer = Diaspora::Markdownify::HTML.new(render_options)
+    if render_options[:oembed]
+      renderer = Diaspora::Markdownify::HTMLwithOEmbed.new(render_options)
+    else
+      renderer = Diaspora::Markdownify::HTML.new(render_options)
+    end
     markdown = Redcarpet::Markdown.new(renderer, markdown_options)
 
     message = markdown.render(message).html_safe

app/models/jobs/gather_o_embed_data.rb

+#   Copyright (c) 2010-2011, Diaspora Inc.  This file is
+#   licensed under the Affero General Public License version 3 or later.  See
+#   the COPYRIGHT file.
+#
+
+module Jobs
+  class GatherOEmbedData < Base
+    @queue = :http_service
+
+    class GatherOEmbedRenderer < Redcarpet::Render::HTML
+      include ActionView::Helpers::TextHelper
+      include ActionView::Helpers::TagHelper
+
+      def autolink(link, type)
+        return link if OEmbedCache.exists?(:url => link)
+        
+        begin
+          res = ::OEmbed::Providers.get(link, {:maxwidth => 420, :maxheight => 420, :frame => 1, :iframe => 1})
+        rescue Exception => e
+          # noop
+        else
+          data = res.fields
+          data['trusted_endpoint_url'] = res.provider.endpoint
+          cache = OEmbedCache.new(:url => link, :data => data)
+          cache.save
+        end
+        
+        return link
+      end
+    end
+
+    def self.perform(text)
+      renderer = GatherOEmbedRenderer.new({})
+      markdown = Redcarpet::Markdown.new(renderer, {:autolink => true})
+      message = markdown.render(text)
+    end
+  end
+end

app/models/o_embed_cache.rb

+class OEmbedCache < ActiveRecord::Base
+  serialize :data
+end

app/models/post.rb

@@ -66,6 +66,11 @@ class Post < ActiveRecord::Base
     @reshare_count ||= Post.where(:root_guid => self.guid).count
   end
 
+  def text= nd
+    Resque.enqueue(Jobs::GatherOEmbedData, nd)
+    write_attribute(:text, nd)
+  end
+
   def diaspora_handle= nd
     self.author = Person.where(:diaspora_handle => nd).first
     write_attribute(:diaspora_handle, nd)

app/views/comments/_comment.html.haml

@@ -12,7 +12,7 @@
       = person_link(comment.author, :class => "hovercardable")
 
     %span{:class => direction_for(comment.text)}
-      = markdownify(comment, :youtube_maps => comment.youtube_titles)
+      = markdownify(comment, :oembed => true, :youtube_maps => comment.youtube_titles)
 
     .comment_info
       %time.timeago{:datetime => comment.created_at}

app/views/messages/_message.haml

@@ -12,5 +12,5 @@
         = how_long_ago(message)
 
     %div{ :class => direction_for(message.text) }
-      = markdownify(message)
+      = markdownify(message, :oembed => true)
 

app/views/people/_profile_sidebar.html.haml

@@ -22,13 +22,13 @@
             %h4
               =t('.bio')
             %div{ :class => direction_for(person.profile.bio) }
-              = markdownify(person.profile.bio, :newlines => true)
+              = markdownify(person.profile.bio, :oembed => true, :newlines => true)
         - unless person.profile.location.blank?
           %li
             %h4
               =t('.location')
             %div{ :class => direction_for(person.profile.location) }
-              = markdownify(person.profile.location, :newlines => true)
+              = markdownify(person.profile.location, :oembed => true, :newlines => true)
 
         %li
           - unless person.profile.gender.blank?

app/views/photos/show.html.haml

@@ -40,7 +40,7 @@
 
     .span-8.last
       %p
-        = markdownify(photo.status_message)
+        = markdownify(photo.status_message, :oembed => true)
       %span{:style=>'font-size:smaller'}
         =link_to t('.collection_permalink'), post_path(photo.status_message)
       %br

app/views/status_messages/_status_message.haml

@@ -16,4 +16,4 @@
           = link_to (image_tag photo.url(:thumb_small), :class => 'stream-photo thumb_small', 'data-small-photo' => photo.url(:thumb_medium), 'data-full-photo' => photo.url), photo_path(photo), :class => 'stream-photo-link'
 
 %div{:class => direction_for(post.text)}
-  != markdownify(post, :youtube_maps => post[:youtube_titles])
+  != markdownify(post, :oembed => true, :youtube_maps => post[:youtube_titles])

config/initializers/oembed.rb

+require 'oembed'
+OEmbed::Providers.register_all
+OEmbed::Providers.register_fallback(OEmbed::ProviderDiscovery)
+

db/migrate/20110924112840_create_o_embed_caches.rb

+class CreateOEmbedCaches < ActiveRecord::Migration
+  def self.up
+    create_table :o_embed_caches do |t|
+      t.string :url, :limit => 1024, :null => false, :unique => true
+      t.text :data, :null => false
+    end
+    add_index :o_embed_caches, :url
+  end
+
+  def self.down
+    remove_index :o_embed_caches, :column => :url
+    drop_table :o_embed_caches
+  end
+end

lib/diaspora/markdownify.rb

@@ -3,12 +3,11 @@ require 'erb'
 module Diaspora
   module Markdownify
     class HTML < Redcarpet::Render::HTML
-
       include ActionView::Helpers::TextHelper
       include ActionView::Helpers::TagHelper
 
       def autolink(link, type)
-        auto_link(link, :link => :urls, :html => { :target => "_blank" })
+        auto_link(link, :link => :urls)
       end
     end
   end

spec/helpers/markdownify_helper_spec.rb

@@ -58,6 +58,134 @@ describe MarkdownifyHelper do
         formatted = markdownify(message)
         formatted.should =~ /hovercard/
       end
+
+      context 'when posting a link with oEmbed support' do
+        scenarios = {
+          "photo" => {
+            "oembed_data" => {
+              "trusted_endpoint_url" => "__!SPOOFED!__",
+              "version" => "1.0",
+              "type" => "photo",
+              "title" => "ZB8T0193",
+              "width" => "240",
+              "height" => "160",
+              "url" => "http://farm4.static.flickr.com/3123/2341623661_7c99f48bbf_m.jpg"
+            },
+            "link_url" => 'http://www.flickr.com/photos/bees/2341623661',
+            "oembed_get_request" => "http://www.flickr.com/services/oembed/?format=json&frame=1&iframe=1&maxheight=420&maxwidth=420&url=http://www.flickr.com/photos/bees/2341623661",
+          },
+
+          "unsupported" => {
+            "oembed_data" => "",
+            "oembed_get_request" => 'http://www.we-do-not-support-oembed.com/index.html',
+            "link_url" => 'http://www.we-do-not-support-oembed.com/index.html'
+          },
+
+          "secure_video" => {
+            "oembed_data" => {
+	            "version" => "1.0",
+	            "type" => "video",
+	            "width" => 425,
+	            "height" => 344,
+	            "title" => "Amazing Nintendo Facts",
+	            "html" => "<object width=\"425\" height=\"344\">
+			            <param name=\"movie\" value=\"http://www.youtube.com/v/M3r2XDceM6A&fs=1\"></param>
+			            <param name=\"allowFullScreen\" value=\"true\"></param>
+			            <param name=\"allowscriptaccess\" value=\"always\"></param>
+			            <embed src=\"http://www.youtube.com/v/M3r2XDceM6A&fs=1\"
+				            type=\"application/x-shockwave-flash\" width=\"425\" height=\"344\"
+				            allowscriptaccess=\"always\" allowfullscreen=\"true\"></embed>
+		            </object>",
+            },
+            "link_url" => "http://youtube.com/watch?v=M3r2XDceM6A&format=json",
+            "oembed_get_request" => "http://www.youtube.com/oembed?format=json&frame=1&iframe=1&maxheight=420&maxwidth=420&url=http://youtube.com/watch?v=M3r2XDceM6A",
+          },
+
+          "unsecure_video" => {
+            "oembed_data" => {
+	            "version" => "1.0",
+	            "type" => "video",
+	            "title" => "This is a video from an unsecure source",
+	            "html" => "<object width=\"425\" height=\"344\">
+			            <param name=\"movie\" value=\"http://www.youtube.com/v/M3r2XDceM6A&fs=1\"></param>
+			            <param name=\"allowFullScreen\" value=\"true\"></param>
+			            <param name=\"allowscriptaccess\" value=\"always\"></param>
+			            <embed src=\"http://www.youtube.com/v/M3r2XDceM6A&fs=1\"
+				            type=\"application/x-shockwave-flash\" width=\"425\" height=\"344\"
+				            allowscriptaccess=\"always\" allowfullscreen=\"true\"></embed>
+		            </object>",
+            },
+            "link_url" => "http://mytube.com/watch?v=M3r2XDceM6A&format=json",
+            "discovery_data" => '<link rel="alternate" type="application/json+oembed" href="http://www.mytube.com/oembed?format=json&frame=1&iframe=1&maxheight=420&maxwidth=420&url=http://mytube.com/watch?v=M3r2XDceM6A" />',
+            "oembed_get_request" => "http://www.mytube.com/oembed?format=json&frame=1&iframe=1&maxheight=420&maxwidth=420&url=http://mytube.com/watch?v=M3r2XDceM6A",
+          },
+
+          "secure_rich" => {
+            "oembed_data" => {
+	            "version" => "1.0",
+	            "type" => "rich",
+	            "width" => 425,
+	            "height" => 344,
+	            "title" => "Amazing Nintendo Facts",
+	            "html" => "<object width=\"425\" height=\"344\">
+			            <param name=\"movie\" value=\"http://www.youtube.com/v/M3r2XDceM6A&fs=1\"></param>
+			            <param name=\"allowFullScreen\" value=\"true\"></param>
+			            <param name=\"allowscriptaccess\" value=\"always\"></param>
+			            <embed src=\"http://www.youtube.com/v/M3r2XDceM6A&fs=1\"
+				            type=\"application/x-shockwave-flash\" width=\"425\" height=\"344\"
+				            allowscriptaccess=\"always\" allowfullscreen=\"true\"></embed>
+		            </object>",
+            },
+            "link_url" => "http://youtube.com/watch?v=M3r2XDceM6A&format=json",
+            "oembed_get_request" => "http://www.youtube.com/oembed?format=json&frame=1&iframe=1&maxheight=420&maxwidth=420&url=http://youtube.com/watch?v=M3r2XDceM6A",
+          },
+
+          "unsecure_rich" => {
+            "oembed_data" => {
+	            "version" => "1.0",
+	            "type" => "rich",
+	            "title" => "This is a video from an unsecure source",
+	            "html" => "<object width=\"425\" height=\"344\">
+			            <param name=\"movie\" value=\"http://www.youtube.com/v/M3r2XDceM6A&fs=1\"></param>
+			            <param name=\"allowFullScreen\" value=\"true\"></param>
+			            <param name=\"allowscriptaccess\" value=\"always\"></param>
+			            <embed src=\"http://www.youtube.com/v/M3r2XDceM6A&fs=1\"
+				            type=\"application/x-shockwave-flash\" width=\"425\" height=\"344\"
+				            allowscriptaccess=\"always\" allowfullscreen=\"true\"></embed>
+		            </object>",
+            },
+            "link_url" => "http://mytube.com/watch?v=M3r2XDceM6A&format=json",
+            "discovery_data" => '<link rel="alternate" type="application/json+oembed" href="http://www.mytube.com/oembed?format=json&frame=1&iframe=1&maxheight=420&maxwidth=420&url=http://mytube.com/watch?v=M3r2XDceM6A" />',
+            "oembed_get_request" => "http://www.mytube.com/oembed?format=json&frame=1&iframe=1&maxheight=420&maxwidth=420&url=http://mytube.com/watch?v=M3r2XDceM6A",
+          },
+        }
+
+        scenarios.each do |type, data|
+          specify 'for type "'+type+'"' do
+            stub_request(:get, data['oembed_get_request']).to_return(:status => 200, :body => data['oembed_data'].to_json.to_s)
+            stub_request(:get, data['link_url']).to_return(:status => 200, :body => data['discovery_data']) if data.has_key?('discovery_data')
+
+            message = "Look at this! "+data['link_url']
+            Jobs::GatherOEmbedData.perform(message)
+            OEmbedCache.find_by_url(data['link_url']).should_not be_nil unless type == 'unsupported'
+
+            formatted = markdownify(message, :oembed => true)
+            case type
+              when 'photo'
+                formatted.should =~ /#{data['oembed_data']['url']}/
+              when 'unsupported'
+                formatted.should =~ /#{data['link_url']}/
+              when 'secure_video', 'secure_rich'
+                formatted.should =~ /#{data['oembed_data']['html']}/
+              when 'unsecure_video', 'unsecure_rich'
+                formatted.should_not =~ /#{data['oembed_data']['html']}/
+                formatted.should =~ /#{data['oembed_data']['title']}/
+                formatted.should =~ /#{data['oembed_data']['url']}/
+            end
+          end
+        end
+
+      end
     end
   end
 end

spec/models/jobs/gather_o_embed_data.rb

+require 'spec_helper'
+describe Jobs::GatherOEmbedData do
+  before do
+    @flickr_oembed_data = {
+      "trusted_endpoint_url" => "__!SPOOFED!__",
+      "version" => "1.0",
+      "type" => "photo",
+      "author_url" => "http://www.flickr.com/photos/bees/",
+      "cache_age" => 3600,
+      "provider_name" => "Flickr",
+      "provider_url" => "http://www.flickr.com/",
+      "title" => "ZB8T0193",
+      "author_name" => "Bees",
+      "width" => "240",
+      "height" => "160",
+      "url" => "https://farm4.static.flickr.com/3123/2341623661_7c99f48bbf_m.jpg"
+    }
+
+    @flickr_oembed_url = 'http://www.flickr.com/services/oembed/'
+    @flickr_photo_url = 'http://www.flickr.com/photos/bees/2341623661'
+    @flickr_oembed_get_request = @flickr_oembed_url+"?format=json&frame=1&iframe=1&maxheight=420&maxwidth=420&url="+@flickr_photo_url
+
+    @no_oembed_url = 'http://www.we-do-not-support-oembed.com/index.html'
+
+    stub_request(:get, @flickr_oembed_get_request).to_return(:status => 200, :body => @flickr_oembed_data.to_json)
+    stub_request(:get, @no_oembed_url).to_return(:status => 200, :body => '<html><body>hello there</body></html>')
+  end
+
+  describe '.perform' do
+    it 'requests not data from the internet' do
+      Jobs::GatherOEmbedData.perform("Look at this! "+@flickr_photo_url)
+
+      a_request(:get, @flickr_oembed_get_request).should have_been_made
+    end
+
+    it 'requests not data from the internet only once' do
+      Jobs::GatherOEmbedData.perform("Look at this! "+@flickr_photo_url)
+      Jobs::GatherOEmbedData.perform("Look at this! "+@flickr_photo_url)
+
+      a_request(:get, @flickr_oembed_get_request).should have_been_made.times(1)
+    end
+
+    it 'creates one cache entry' do
+      Jobs::GatherOEmbedData.perform("Look at this! "+@flickr_photo_url)
+
+      expected_data = @flickr_oembed_data
+      expected_data['trusted_endpoint_url'] = @flickr_oembed_url
+      OEmbedCache.find_by_url(@flickr_photo_url).data.should == expected_data
+
+      Jobs::GatherOEmbedData.perform("Look at this! "+@flickr_photo_url)
+      OEmbedCache.count(:conditions => {:url => @flickr_photo_url}).should == 1
+    end
+
+    it 'creates no cache entry for unsupported pages' do
+      Jobs::GatherOEmbedData.perform("This page is lame! It does not support oEmbed: "+@no_oembed_url)
+      OEmbedCache.find_by_url(@no_oembed_url).should be_nil
+    end
+  end
+end

spec/models/o_embed_cache_spec.rb

+require 'spec_helper'
+
+describe OEmbedCache do
+  pending "add some examples to (or delete) #{__FILE__}"
+end

spec/models/post_spec.rb

@@ -68,6 +68,17 @@ describe Post do
     end
   end
 
+  describe 'oEmbed' do
+    it 'should gather oEmbed data' do
+      stub_request(:get, "http://gdata.youtube.com/feeds/api/videos/M3r2XDceM6A?v=2").to_return(:status => 200, :body => "")
+
+      text = 'http://youtube.com/watch?v=M3r2XDceM6A&format=json'
+      Resque.should_receive(:enqueue).with(Jobs::GatherOEmbedData, text)
+      post = Factory.create(:status_message, :author => @user.person, :text => text)
+      post.save!
+    end
+  end
+
   describe 'serialization' do
     it 'should serialize the handle and not the sender' do
       post = @user.post :status_message, :text => "hello", :to => @aspect.id