From e709b8da0d685d3cc48d430a9761896094f67d72 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 17 Dec 2018 19:19:45 +0100
Subject: [PATCH] Ignore low-confidence CharlockHolmes guesses when parsing
link cards (#9510)
* Add failing test for windows-1251 link cards
* Ignore low-confidence CharlockHolmes guesses
Fixes #9466
* Fix no method error when charlock holmes cannot detect charset
---
app/services/fetch_link_card_service.rb | 3 ++-
spec/fixtures/requests/windows-1251.txt | 17 +++++++++++++++++
spec/services/fetch_link_card_service_spec.rb | 11 +++++++++++
3 files changed, 30 insertions(+), 1 deletion(-)
create mode 100644 spec/fixtures/requests/windows-1251.txt
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 38c578de2..7979c312e 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -137,7 +137,8 @@ class FetchLinkCardService < BaseService
detector.strip_tags = true
guess = detector.detect(@html, @html_charset)
- page = Nokogiri::HTML(@html, nil, guess&.fetch(:encoding, nil))
+ encoding = guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
+ page = Nokogiri::HTML(@html, nil, encoding)
player_url = meta_property(page, 'twitter:player')
if player_url && !bad_url?(Addressable::URI.parse(player_url))
diff --git a/spec/fixtures/requests/windows-1251.txt b/spec/fixtures/requests/windows-1251.txt
new file mode 100644
index 000000000..f573e28b2
--- /dev/null
+++ b/spec/fixtures/requests/windows-1251.txt
@@ -0,0 +1,17 @@
+HTTP/1.1 200 OK
+server: nginx
+date: Wed, 12 Dec 2018 13:14:03 GMT
+content-type: text/html
+content-length: 190
+accept-ranges: bytes
+
+<!DOCTYPE html>
+<html>
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=windows-1251" />
+ <title>����� �����</title>
+</head>
+<body>
+ <p>����� �����</p>
+</body>
+</html>
diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb
index 88c5339db..50c60aafd 100644
--- a/spec/services/fetch_link_card_service_spec.rb
+++ b/spec/services/fetch_link_card_service_spec.rb
@@ -17,6 +17,8 @@ RSpec.describe FetchLinkCardService, type: :service do
stub_request(:head, 'https://github.com/qbi/WannaCry').to_return(status: 404)
stub_request(:head, 'http://example.com/test-').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
stub_request(:get, 'http://example.com/test-').to_return(request_fixture('idn.txt'))
+ stub_request(:head, 'http://example.com/windows-1251').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
+ stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt'))
subject.call(status)
end
@@ -57,6 +59,15 @@ RSpec.describe FetchLinkCardService, type: :service do
end
end
+ context do
+ let(:status) { Fabricate(:status, text: 'Check out http://example.com/windows-1251') }
+
+ it 'works with windows-1251' do
+ expect(a_request(:get, 'http://example.com/windows-1251')).to have_been_made.at_least_once
+ expect(status.preview_cards.first.title).to eq('сэмпл текст')
+ end
+ end
+
context do
let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') }
--
GitLab