Initial Commit

Author: mattt

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "data"]
+	path = data
+	url = [email protected]:mattt/wwdc-session-transcripts.git

Gemfile

@@ -0,0 +1,22 @@
+source "https://rubygems.org"
+ruby "2.0.0"
+
+gem 'pg'
+gem 'sequel'
+
+gem 'rake'
+
+gem 'rack-gauges', require: 'rack/gauges'
+
+gem 'sinatra', require: 'sinatra/base'
+gem 'sinatra-contrib', require: 'sinatra/contrib/all'
+gem 'sinatra-param', require: 'sinatra/param'
+
+gem 'haml'
+gem 'redcarpet'
+
+gem 'unicorn'
+
+group :development do
+  gem 'compass'
+end

Gemfile.lock

@@ -0,0 +1,60 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    backports (3.3.3)
+    chunky_png (1.2.8)
+    compass (0.12.2)
+      chunky_png (~> 1.2)
+      fssm (>= 0.2.7)
+      sass (~> 3.1)
+    fssm (0.2.10)
+    haml (4.0.3)
+      tilt
+    kgio (2.8.1)
+    multi_json (1.7.9)
+    pg (0.17.0)
+    rack (1.5.2)
+    rack-gauges (1.1.0)
+    rack-protection (1.5.0)
+      rack
+    rack-test (0.6.2)
+      rack (>= 1.0)
+    raindrops (0.12.0)
+    rake (10.1.0)
+    redcarpet (3.0.0)
+    sass (3.2.10)
+    sequel (4.2.0)
+    sinatra (1.4.3)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
+    sinatra-contrib (1.4.1)
+      backports (>= 2.0)
+      multi_json
+      rack-protection
+      rack-test
+      sinatra (~> 1.4.0)
+      tilt (~> 1.3)
+    sinatra-param (0.1.3)
+      sinatra (~> 1.3)
+    tilt (1.4.1)
+    unicorn (4.6.3)
+      kgio (~> 2.6)
+      rack
+      raindrops (~> 0.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  compass
+  haml
+  pg
+  rack-gauges
+  rake
+  redcarpet
+  sequel
+  sinatra
+  sinatra-contrib
+  sinatra-param
+  unicorn

Procfile

@@ -0,0 +1,2 @@
+web: bundle exec unicorn -p $PORT
+compass: bundle exec compass watch public

README.md

@@ -0,0 +1,2 @@
+# ASCIIwwdc
+*Searchable full-text transcripts of WWDC sessions*

Rakefile

@@ -0,0 +1,37 @@
+require 'bundler'
+Bundler.require
+
+require 'yaml'
+
+Sequel.extension :migration
+
+DB = Sequel.connect(ENV['DATABASE_URL'])
+Sequel::Migrator.run(DB, ::File.join(::File.dirname(__FILE__), 'lib/migrations'))
+
+require './lib/models/session'
+
+namespace :db do
+  task :seed do
+    Dir["data/*"].each do |directory|
+      year = Integer(directory.split(/\//).last)
+
+      YAML.load(File.open(File.join(directory, "_sessions.yml"))).each do |number, attributes|
+        session = Session.new(attributes)
+        session.number = number
+        session.year = year
+        session.transcript = File.read("data/#{year}/#{number}.srt").lines.delete_if{|line| 
+          line == "\n" || 
+          line[0] == "[" || 
+          /^\d{2}\:\d{2}\:\d{2}\.\d{3}/ === line || 
+          /^WEBVTT/ === line || 
+          /^X-TIMESTAMP-MAP/ === line
+        }.collect{|line| 
+          line.gsub(/[\r\n]+/, " ").gsub(/(&gt\;|\-\-)/, "")
+        }.join
+
+        puts session
+        session.save
+      end
+    end
+  end
+end

config.ru

@@ -0,0 +1,16 @@
+require 'bundler'
+Bundler.require
+
+DB = Sequel.connect(ENV['DATABASE_URL'])
+
+Rack::Mime::MIME_TYPES.merge!({
+  ".srt" => "text/plain",
+  ".vtt" => "text/vtt",
+})
+
+use Rack::Static, urls: ["/css", "/images", "/js", "favicon.ico"], root: "public"
+use Rack::Gauges, tracker: ENV['GAUGES_TRACKER_ID'] if ENV['GAUGES_TRACKER_ID']
+
+require './web'
+
+run Web

data

@@ -0,0 +1,20 @@
+Sequel.migration do
+  up do
+    create_table :sessions do
+      primary_key :id
+      
+      varchar   :title,       empty: false
+      text      :description
+      text      :transcript
+      text      :markup
+      integer   :year,        null: false
+      integer   :number,      null: false
+      varchar   :track
+      interval  :duration
+    end
+  end
+  
+  down do
+    drop_table :sessions
+  end
+end

lib/migrations/001_base_schema.rb

@@ -0,0 +1,16 @@
+Sequel.migration do
+  up do
+    add_column :sessions, :tsv, 'TSVector'
+    add_index :sessions, :tsv, type: "GIN"
+    create_trigger :sessions, :tsv, :tsvector_update_trigger,
+      args: [:tsv, :'pg_catalog.english', :transcript],
+      events: [:insert, :update],
+      each_row: true
+  end
+
+  down do
+    drop_column :sessions, :tsv
+    drop_index :sessions, :tsv
+    drop_trigger :sessions, :tsv
+  end
+end

lib/migrations/002_add_full_text_index.rb

@@ -0,0 +1,56 @@
+require 'ostruct'
+
+class Session < Sequel::Model
+  plugin :json_serializer, naked: true, except: [:id, :tsv]
+  plugin :validation_helpers
+  plugin :schema
+
+  def self.search(query)
+    query = Session.db.literal(query.to_s + ":*")
+
+    Session.db[%{
+      SELECT title, description, year, number, 
+             ts_rank_cd(
+              (
+                setweight(to_tsvector(title), 'A') || 
+                setweight(to_tsvector(description), 'C') || 
+                setweight(tsv, 'D')
+              ),
+              plainto_tsquery('english', #{query})
+             ) AS rank, 
+             ts_headline(
+              'pg_catalog.english', transcript, plainto_tsquery('english', #{query}), 
+              'ShortWord=0, MinWords=50, MaxWords=70'
+             ) AS excerpt 
+             FROM sessions 
+             WHERE tsv @@ plainto_tsquery('english', #{query}) 
+             ORDER BY rank DESC
+    }].collect{|result| OpenStruct.new(result)}
+  end
+
+  def to_s
+    "#{self.number} #{self.title}"
+  end
+
+  def validate
+    super
+
+    validates_presence [:title, :description, :year, :track, :transcript]
+    validates_unique [:title, :year]
+    validates_unique [:number, :year]
+  end
+
+  def before_save
+    super
+
+    html = ""
+
+    previous = nil
+    self.transcript.split(/\.\s+/).each do |sentence|
+      html << "<p>#{sentence.strip}.</p>" unless sentence == previous
+      previous = sentence
+    end
+
+    self.markup = html
+  end
+end

lib/models/session.rb

@@ -0,0 +1,9 @@
+http_path = "/"
+css_dir = "css"
+sass_dir = "sass"
+images_dir = "images"
+javascripts_dir = "js"
+
+output_style = :compressed
+line_comments = false
+preferred_syntax = :sass