Track trending tags (#7638)

* Track trending tags

- Half-life of 1 day
- Historical usage in daily buckets (last 7 days stored)
- GET /api/v1/trends

Fix #271

* Add trends to web UI

* Don't render compose form on search route, adjust search results header

* Disqualify tag from trends if it's in disallowed hashtags setting

* Count distinct accounts using tag, ignore silenced accounts

app/controllers/api/v1/trends_controller.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class Api::V1::TrendsController < Api::BaseController
+  before_action :set_tags
+
+  respond_to :json
+
+  def index
+    render json: @tags, each_serializer: REST::TagSerializer
+  end
+
+  private
+
+  def set_tags
+    @tags = TrendingTags.get(limit_param(10))
+  end
+end

app/javascript/mastodon/actions/trends.js

@@ -0,0 +1,32 @@
+import api from '../api';
+
+export const TRENDS_FETCH_REQUEST = 'TRENDS_FETCH_REQUEST';
+export const TRENDS_FETCH_SUCCESS = 'TRENDS_FETCH_SUCCESS';
+export const TRENDS_FETCH_FAIL    = 'TRENDS_FETCH_FAIL';
+
+export const fetchTrends = () => (dispatch, getState) => {
+  dispatch(fetchTrendsRequest());
+
+  api(getState)
+    .get('/api/v1/trends')
+    .then(({ data }) => dispatch(fetchTrendsSuccess(data)))
+    .catch(err => dispatch(fetchTrendsFail(err)));
+};
+
+export const fetchTrendsRequest = () => ({
+  type: TRENDS_FETCH_REQUEST,
+  skipLoading: true,
+});
+
+export const fetchTrendsSuccess = trends => ({
+  type: TRENDS_FETCH_SUCCESS,
+  trends,
+  skipLoading: true,
+});
+
+export const fetchTrendsFail = error => ({
+  type: TRENDS_FETCH_FAIL,
+  error,
+  skipLoading: true,
+  skipAlert: true,
+});

app/javascript/mastodon/features/compose/components/search_results.js

@@ -1,23 +1,75 @@
 import React from 'react';
+import PropTypes from 'prop-types';
 import ImmutablePropTypes from 'react-immutable-proptypes';
-import { FormattedMessage } from 'react-intl';
+import { FormattedMessage, FormattedNumber } from 'react-intl';
 import AccountContainer from '../../../containers/account_container';
 import StatusContainer from '../../../containers/status_container';
 import { Link } from 'react-router-dom';
 import ImmutablePureComponent from 'react-immutable-pure-component';
+import { Sparklines, SparklinesCurve } from 'react-sparklines';
+
+const shortNumberFormat = number => {
+  if (number < 1000) {
+    return <FormattedNumber value={number} />;
+  } else {
+    return <React.Fragment><FormattedNumber value={number / 1000} maximumFractionDigits={1} />K</React.Fragment>;
+  }
+};
 
 export default class SearchResults extends ImmutablePureComponent {
 
   static propTypes = {
     results: ImmutablePropTypes.map.isRequired,
+    trends: ImmutablePropTypes.list,
+    fetchTrends: PropTypes.func.isRequired,
   };
 
+  componentDidMount () {
+    const { fetchTrends } = this.props;
+    fetchTrends();
+  }
+
   render () {
-    const { results } = this.props;
+    const { results, trends } = this.props;
 
     let accounts, statuses, hashtags;
     let count = 0;
 
+    if (results.isEmpty()) {
+      return (
+        <div className='search-results'>
+          <div className='trends'>
+            <div className='trends__header'>
+              <i className='fa fa-fire fa-fw' />
+              <FormattedMessage id='trends.header' defaultMessage='Trending now' />
+            </div>
+
+            {trends && trends.map(hashtag => (
+              <div className='trends__item' key={hashtag.get('name')}>
+                <div className='trends__item__name'>
+                  <Link to={`/timelines/tag/${hashtag.get('name')}`}>
+                    #<span>{hashtag.get('name')}</span>
+                  </Link>
+
+                  <FormattedMessage id='trends.count_by_accounts' defaultMessage='{count} {rawCount, plural, one {person} other {people}} talking' values={{ rawCount: hashtag.getIn(['history', 0, 'accounts']), count: <strong>{shortNumberFormat(hashtag.getIn(['history', 0, 'accounts']))}</strong> }} />
+                </div>
+
+                <div className='trends__item__current'>
+                  {shortNumberFormat(hashtag.getIn(['history', 0, 'uses']))}
+                </div>
+
+                <div className='trends__item__sparkline'>
+                  <Sparklines width={50} height={28} data={hashtag.get('history').reverse().map(day => day.get('uses')).toArray()}>
+                    <SparklinesCurve style={{ fill: 'none' }} />
+                  </Sparklines>
+                </div>
+              </div>
+            ))}
+          </div>
+        </div>
+      );
+    }
+
     if (results.get('accounts') && results.get('accounts').size > 0) {
       count   += results.get('accounts').size;
       accounts = (
@@ -48,7 +100,7 @@ export default class SearchResults extends ImmutablePureComponent {
 
           {results.get('hashtags').map(hashtag => (
             <Link key={hashtag} className='search-results__hashtag' to={`/timelines/tag/${hashtag}`}>
-              #{hashtag}
+              {hashtag}
             </Link>
           ))}
         </div>
@@ -58,6 +110,7 @@ export default class SearchResults extends ImmutablePureComponent {
     return (
       <div className='search-results'>
         <div className='search-results__header'>
+          <i className='fa fa-search fa-fw' />
           <FormattedMessage id='search_results.total' defaultMessage='{count, number} {count, plural, one {result} other {results}}' values={{ count }} />
         </div>
 

app/javascript/mastodon/features/compose/containers/search_results_container.js

@@ -1,8 +1,14 @@
 import { connect } from 'react-redux';
 import SearchResults from '../components/search_results';
+import { fetchTrends } from '../../../actions/trends';
 
 const mapStateToProps = state => ({
   results: state.getIn(['search', 'results']),
+  trends: state.get('trends'),
 });
 
-export default connect(mapStateToProps)(SearchResults);
+const mapDispatchToProps = dispatch => ({
+  fetchTrends: () => dispatch(fetchTrends()),
+});
+
+export default connect(mapStateToProps, mapDispatchToProps)(SearchResults);

app/javascript/mastodon/features/compose/index.js

@@ -101,7 +101,7 @@ export default class Compose extends React.PureComponent {
         {(multiColumn || isSearchPage) && <SearchContainer /> }
 
         <div className='drawer__pager'>
-          <div className='drawer__inner' onFocus={this.onFocus}>
+          {!isSearchPage && <div className='drawer__inner' onFocus={this.onFocus}>
             <NavigationContainer onClose={this.onBlur} />
             <ComposeFormContainer />
             {multiColumn && (
@@ -109,7 +109,7 @@ export default class Compose extends React.PureComponent {
                 <img alt='' draggable='false' src={elephantUIPlane} />
               </div>
             )}
-          </div>
+          </div>}
 
           <Motion defaultStyle={{ x: isSearchPage ? 0 : -100 }} style={{ x: spring(showSearch || isSearchPage ? 0 : -100, { stiffness: 210, damping: 20 }) }}>
             {({ x }) => (

app/javascript/mastodon/reducers/index.js

@@ -26,6 +26,7 @@ import height_cache from './height_cache';
 import custom_emojis from './custom_emojis';
 import lists from './lists';
 import listEditor from './list_editor';
+import trends from './trends';
 
 const reducers = {
   dropdown_menu,
@@ -55,6 +56,7 @@ const reducers = {
   custom_emojis,
   lists,
   listEditor,
+  trends,
 };
 
 export default combineReducers(reducers);

app/javascript/mastodon/reducers/trends.js

@@ -0,0 +1,13 @@
+import { TRENDS_FETCH_SUCCESS } from '../actions/trends';
+import { fromJS } from 'immutable';
+
+const initialState = null;
+
+export default function trendsReducer(state = initialState, action) {
+  switch(action.type) {
+  case TRENDS_FETCH_SUCCESS:
+    return fromJS(action.trends);
+  default:
+    return state;
+  }
+};

app/javascript/styles/mastodon/components.scss

@@ -3334,9 +3334,15 @@ a.status-card {
   color: $dark-text-color;
   background: lighten($ui-base-color, 2%);
   border-bottom: 1px solid darken($ui-base-color, 4%);
-  padding: 15px 10px;
+  padding: 15px;
-  font-size: 14px;
   font-weight: 500;
+  font-size: 16px;
+  cursor: default;
+
+  .fa {
+    display: inline-block;
+    margin-right: 5px;
+  }
 }
 
 .search-results__section {
@@ -5209,3 +5215,76 @@ noscript {
     background: $ui-base-color;
   }
 }
+
+.trends {
+  &__header {
+    color: $dark-text-color;
+    background: lighten($ui-base-color, 2%);
+    border-bottom: 1px solid darken($ui-base-color, 4%);
+    font-weight: 500;
+    padding: 15px;
+    font-size: 16px;
+    cursor: default;
+
+    .fa {
+      display: inline-block;
+      margin-right: 5px;
+    }
+  }
+
+  &__item {
+    display: flex;
+    align-items: center;
+    padding: 15px;
+    border-bottom: 1px solid lighten($ui-base-color, 8%);
+
+    &:last-child {
+      border-bottom: 0;
+    }
+
+    &__name {
+      flex: 1 1 auto;
+      color: $dark-text-color;
+      overflow: hidden;
+      text-overflow: ellipsis;
+      white-space: nowrap;
+
+      strong {
+        font-weight: 500;
+      }
+
+      a {
+        color: $darker-text-color;
+        text-decoration: none;
+        font-size: 14px;
+        font-weight: 500;
+        display: block;
+
+        &:hover,
+        &:focus,
+        &:active {
+          span {
+            text-decoration: underline;
+          }
+        }
+      }
+    }
+
+    &__current {
+      width: 100px;
+      font-size: 24px;
+      line-height: 36px;
+      font-weight: 500;
+      text-align: center;
+      color: $secondary-text-color;
+    }
+
+    &__sparkline {
+      width: 50px;
+
+      path {
+        stroke: lighten($highlight-text-color, 6%) !important;
+      }
+    }
+  }
+}

app/models/tag.rb

@@ -21,6 +21,22 @@ class Tag < ApplicationRecord
     name
   end
 
+  def history
+    days = []
+
+    7.times do |i|
+      day = i.days.ago.beginning_of_day.to_i
+
+      days << {
+        day: day.to_s,
+        uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0',
+        accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s,
+      }
+    end
+
+    days
+  end
+
   class << self
     def search_for(term, limit = 5)
       pattern = sanitize_sql_like(term.strip) + '%'

app/models/trending_tags.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+class TrendingTags
+  KEY                  = 'trending_tags'
+  HALF_LIFE            = 1.day.to_i
+  MAX_ITEMS            = 500
+  EXPIRE_HISTORY_AFTER = 7.days.seconds
+
+  class << self
+    def record_use!(tag, account, at_time = Time.now.utc)
+      return if disallowed_hashtags.include?(tag.name) || account.silenced?
+
+      increment_vote!(tag.id, at_time)
+      increment_historical_use!(tag.id, at_time)
+      increment_unique_use!(tag.id, account.id, at_time)
+    end
+
+    def get(limit)
+      tag_ids = redis.zrevrange(KEY, 0, limit).map(&:to_i)
+      tags    = Tag.where(id: tag_ids).to_a.map { |tag| [tag.id, tag] }.to_h
+      tag_ids.map { |tag_id| tags[tag_id] }.compact
+    end
+
+    private
+
+    def increment_vote!(tag_id, at_time)
+      redis.zincrby(KEY, (2**((at_time.to_i - epoch) / HALF_LIFE)).to_f, tag_id.to_s)
+      redis.zremrangebyrank(KEY, 0, -MAX_ITEMS) if rand < (2.to_f / MAX_ITEMS)
+    end
+
+    def increment_historical_use!(tag_id, at_time)
+      key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}"
+      redis.incrby(key, 1)
+      redis.expire(key, EXPIRE_HISTORY_AFTER)
+    end
+
+    def increment_unique_use!(tag_id, account_id, at_time)
+      key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts"
+      redis.pfadd(key, account_id)
+      redis.expire(key, EXPIRE_HISTORY_AFTER)
+    end
+
+    # The epoch needs to be 2.5 years in the future if the half-life is one day
+    # While dynamic, it will always be the same within one year
+    def epoch
+      @epoch ||= Date.new(Date.current.year + 2.5, 10, 1).to_datetime.to_i
+    end
+
+    def disallowed_hashtags
+      return @disallowed_hashtags if defined?(@disallowed_hashtags)
+
+      @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
+      @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
+      @disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
+    end
+
+    def redis
+      Redis.current
+    end
+  end
+end

app/serializers/rest/tag_serializer.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class REST::TagSerializer < ActiveModel::Serializer
+  include RoutingHelper
+
+  attributes :name, :url, :history
+
+  def url
+    tag_url(object)
+  end
+end

app/services/process_hashtags_service.rb

@@ -4,8 +4,10 @@ class ProcessHashtagsService < BaseService
   def call(status, tags = [])
     tags = Extractor.extract_hashtags(status.text) if status.local?
 
-    tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |tag|
+    tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |name|
-      status.tags << Tag.where(name: tag).first_or_initialize(name: tag)
+      tag = Tag.where(name: name).first_or_create(name: name)
+      status.tags << tag
+      TrendingTags.record_use!(tag, status.account, status.created_at)
     end
   end
 end

config/routes.rb

@@ -254,6 +254,7 @@ Rails.application.routes.draw do
       resources :mutes,      only: [:index]
       resources :favourites, only: [:index]
       resources :reports,    only: [:index, :create]
+      resources :trends,     only: [:index]
 
       namespace :apps do
         get :verify_credentials, to: 'credentials#show'

package.json

@@ -97,6 +97,7 @@
     "react-redux-loading-bar": "^2.9.3",
     "react-router-dom": "^4.1.1",
     "react-router-scroll-4": "^1.0.0-beta.1",
+    "react-sparklines": "^1.7.0",
     "react-swipeable-views": "^0.12.3",
     "react-textarea-autosize": "^5.2.1",
     "react-toggle": "^4.0.1",

yarn.lock

@@ -6124,6 +6124,12 @@ react-router@^4.2.0:
     prop-types "^15.5.4"
     warning "^3.0.0"
 
+react-sparklines@^1.7.0:
+  version "1.7.0"
+  resolved "https://registry.yarnpkg.com/react-sparklines/-/react-sparklines-1.7.0.tgz#9b1d97e8c8610095eeb2ad658d2e1fcf91f91a60"
+  dependencies:
+    prop-types "^15.5.10"
+
 react-swipeable-views-core@^0.12.11:
   version "0.12.11"
   resolved "https://registry.yarnpkg.com/react-swipeable-views-core/-/react-swipeable-views-core-0.12.11.tgz#3cf2b4daffbb36f9d69bd19bf5b2d5370b6b2c1b"