.gitignore

@@ -6,3 +6,8 @@ clojure/cache
 *.pyc
 planet/*.pyc
 *~
+/twitter-data.txt
+/debug.atom
+/fetch.log
+/t1.py
+/posted_urls.pickle

README.md

@@ -24,8 +24,20 @@ After you are done, commit the change to your repository and send me a
 pull request. I will be happy to add you to Planet Clojure.
 
 Note: Please add the feed which contains only those posts which are 
-Clojure/Lisp related. As a policy we do not put generic feeds on Planet 
-Clojure. 
+Clojure/Lisp related. 
+
+As a policy we do not put generic feeds on Planet Clojure, but if you don't have
+Clojure-specific feed on your site, you may try to add following parameter to the section
+to filter only Clojure-specific posts (but be careful, it may not be very precise if your
+feed doesn't include enough text to match this regex):
+
+    filter = (clojure|Clojure|\(def |\(defn-? )
+
+If you also have Twitter account, please add `twitter` parameter to the section, like
+this:
+
+    twitter = <handle-without-@>
+
 
 Reporting Bugs
 --------------

clojure/index.html.tmpl

@@ -12,8 +12,16 @@
 <script type="text/javascript" src="static/js/html5.js"></script>
 <![endif]-->
 
-    <link rel="icon" type="image/vnd.microsoft.icon" href="static/i/favicon.gif" />
-    <link rel="shortcut icon" href="static/i/favicon.gif" />
+
+
+    <link rel="apple-touch-icon" sizes="180x180" href="/static/i/apple-touch-icon.png">
+    <link rel="icon" type="image/png" sizes="32x32" href="/static/i/favicon-32x32.png">
+    <link rel="icon" type="image/png" sizes="16x16" href="/static/i/favicon-16x16.png">
+    <link rel="manifest" href="/static/i/site.webmanifest">
+    <link rel="shortcut icon" href="/static/i/favicon.ico">
+    <meta name="msapplication-TileColor" content="#da532c">
+    <meta name="msapplication-config" content="/static/i/browserconfig.xml">
+    <meta name="theme-color" content="#ffffff">
     <link rel="alternate" href="http://planet.clojure.in/atom.xml" type="application/rss+xml" title="Planet Clojure RSS Feed" />
 <!--    <link rel="alternate" href="http://feeds.feedburner.com/clojure" type="application/rss+xml" title="Planet Clojure RSS Feed" />-->
   </head>
@@ -94,14 +102,14 @@
             <h3>Planetarium</h3>
             <ul>
               <li><a href="http://planet.lisp.org/">Planet Lisp</a></li>
-              <li><a href="http://www.planetscala.com/">Planet Scala</a></li>
-              <li><a href="http://scheme.dk/planet/">Planet Scheme</a></li>
-              <li><a href="http://planetjdk.org/">Planet JDK</a></li>
-              <li><a href="http://planet.haskell.org/">Planet Haskell</a></li>
-              <li><a href="http://www.planeterlang.org/">Planet Erlang</a></li>
-              <li><a href="http://planet.python.org/">Planet Python</a></li>
-              <li><a href="http://planetruby.0x42.net/">Planet Ruby</a></li>
-              <li><a href="http://planet.perl.org/">Planet Perl</a></li>
+<!--              <li><a href="http://www.planetscala.com/">Planet Scala</a></li>-->
+              <li><a href="https://planet.scheme.org">Planet Scheme</a></li>
+<!--              <li><a href="http://planetjdk.org/">Planet JDK</a></li>-->
+              <li><a href="https://planet.haskell.org">Planet Haskell</a></li>
+<!--              <li><a href="http://www.planeterlang.org">Planet Erlang</a></li>-->
+              <li><a href="http://planet.python.org">Planet Python</a></li>
+<!--              <li><a href="http://planetruby.0x42.net">Planet Ruby</a></li>-->
+<!--              <li><a href="http://planet.perl.org">Planet Perl</a></li>-->
             </ul>
           </section>
 

clojure/static/i/browserconfig.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<browserconfig>
+    <msapplication>
+        <tile>
+            <square150x150logo src="/static/i/mstile-150x150.png"/>
+            <TileColor>#da532c</TileColor>
+        </tile>
+    </msapplication>
+</browserconfig>

clojure/static/i/site.webmanifest

@@ -0,0 +1,14 @@
+{
+    "name": "",
+    "short_name": "",
+    "icons": [
+        {
+            "src": "/static/i/android-chrome-192x192.png",
+            "sizes": "192x192",
+            "type": "image/png"
+        }
+    ],
+    "theme_color": "#ffffff",
+    "background_color": "#ffffff",
+    "display": "standalone"
+}

clojure/static/sass/_clojure.sass

@@ -25,7 +25,7 @@
         display: inline-block
         width: 160px
         height: 26px
-        +replace-text("plantet-clojure-logo.gif", 100%, 0)
+        +replace-text("planet-clojure-logo.svg", 100%, 0)
 
         &:focus, &:hover
             background-position: 100% -50px            

docs/config.html

@@ -94,6 +94,16 @@ <h3 id="planet"><code>[planet]</code></h3>
 number of days will be marked as inactive</dd>
 
 </dl>
+<dl class="compact code">
+  <dt>post_to_twitter</dt><dd>(<code>true</code> or <code>false</code> (default if
+    omitted)) enables the cross posting of articles to Twitter account defined by
+    following environment variables:
+    <code>twitter_consumer_key</code>, <code>twitter_consumer_secret</code>,
+    <code>twitter_access_token</code>, <code>twitter_access_token_secret</code>.
+    If site's section has <code>twitter</code> configuration, then this handle will be
+    used to specify author's name.</dd>
+</dl>
+
 <dl class="compact code">
 
 <dt>log_level</dt>

filters/nopipeerrors.py

@@ -0,0 +1,6 @@
+#remove all entries with Yahoo! Pipes error
+import sys
+
+data = sys.stdin.read()
+if data.find('No such pipe, or this pipe has been deleted') < 0:
+  sys.stdout.write(data)

filters/regexp_sifter.py

@@ -14,13 +14,12 @@
   (re.compile('<source>.*?</source>'),' '),
   (re.compile('<updated.*?</updated>'),' '),
   (re.compile('<published.*?</published>'),' '),
-  (re.compile('<link .*?>'),' '),
+  (re.compile('<link [^>]*>'),' '),
   (re.compile('''<[^>]* alt=['"]([^'"]*)['"].*?>'''),r' \1 '),
   (re.compile('''<[^>]* title=['"]([^'"]*)['"].*?>'''),r' \1 '),
   (re.compile('''<[^>]* label=['"]([^'"]*)['"].*?>'''),r' \1 '),
   (re.compile('''<[^>]* term=['"]([^'"]*)['"].*?>'''),r' \1 '),
-  (re.compile('<.*?>'),' '),
-  (re.compile('\s+'),' '),
+  (re.compile('<[^>]+>'),' '),
   (re.compile('&gt;'),'>'),
   (re.compile('&lt;'),'<'),
   (re.compile('&apos;'),"'"),

planet/config.py

@@ -70,11 +70,16 @@ def define_planet(name, default):
         setattr(config, name, lambda default=default: get(None,name,default))
         planet_predefined_options.append(name)
 
-    # define a list planet-level variable
+    # define a int planet-level variable
     def define_planet_int(name, default=0):
         setattr(config, name, lambda : int(get(None,name,default)))
         planet_predefined_options.append(name)
 
+    # define a bool planet-level variable
+    def define_planet_bool(name, default='False'):
+        setattr(config, name, lambda : get(None,name,default).lower().strip() == 'true')
+        planet_predefined_options.append(name)
+
     # define a list planet-level variable
     def define_planet_list(name, default=''):
         setattr(config, name, lambda : expand(get(None,name,default)))
@@ -107,6 +112,7 @@ def define_tmpl_int(name, default):
     define_planet('spider_threads', 0) 
     define_planet('pubsubhubbub_hub', '')
     define_planet_list('pubsubhubbub_feeds', 'atom.xml rss10.xml rss20.xml')
+    define_planet_bool('post_to_twitter')
 
     define_planet_int('new_feed_items', 0) 
     define_planet_int('feed_timeout', 20)
@@ -133,6 +139,13 @@ def define_tmpl_int(name, default):
     define_tmpl('filter', None) 
     define_tmpl('exclude', None) 
 
+def get_twitter_conf_value(conf, name):
+    attr = os.getenv(name)
+    if (attr is None):
+	raise Exception('Environment variable "{}" is not set!'.format(name))
+
+    return attr
+
 def load(config_file):
     """ initialize and load a configuration"""
     global parser
@@ -215,6 +228,21 @@ def data2config(data, cached_config):
         for list in reading_lists:
             downloadReadingList(list, parser, data2config)
 
+    # for Twitter integration
+    if config.post_to_twitter():
+	log.info('Twitter integration is enabled')
+	twitter_consumer_key = get_twitter_conf_value(config, 'twitter_consumer_key')
+	twitter_consumer_secret = get_twitter_conf_value(config, 'twitter_consumer_secret')
+	twitter_access_token = get_twitter_conf_value(config, 'twitter_access_token')
+	twitter_access_token_secret = get_twitter_conf_value(config, 'twitter_access_token_secret')
+
+	import tweepy
+	auth = tweepy.OAuthHandler(twitter_consumer_key, twitter_consumer_secret)
+	auth.set_access_token(twitter_access_token, twitter_access_token_secret)
+	api = tweepy.API(auth)
+
+	setattr(config, 'twitter_api', api)
+
 def downloadReadingList(list, orig_config, callback, use_cache=True, re_read=True):
     from planet import logger
     import config
@@ -349,7 +377,7 @@ def feedtype():
 
 def subscriptions():
     """ list the feed subscriptions """
-    return __builtins__['filter'](lambda feed: feed!='Planet' and 
+    return __builtins__['filter'](lambda feed: feed != 'Planet' and 
         feed not in template_files()+filters()+reading_lists(),
         parser.sections())
 

planet/idindex.py

@@ -13,8 +13,8 @@ def open():
         cache = config.cache_directory()
         index=os.path.join(cache,'index')
         if not os.path.exists(index): return None
-        import dbhash
-        return dbhash.open(filename(index, 'id'),'w')
+        import anydbm
+        return anydbm.open(filename(index, 'id'),'w')
     except Exception, e:
         if e.__class__.__name__ == 'DBError': e = e.args[-1]
         from planet import logger as log
@@ -35,8 +35,8 @@ def create():
     cache = config.cache_directory()
     index=os.path.join(cache,'index')
     if not os.path.exists(index): os.makedirs(index)
-    import dbhash
-    index = dbhash.open(filename(index, 'id'),'c')
+    import anydbm
+    index = anydbm.open(filename(index, 'id'),'c')
 
     try:
         import libxml2

planet/spider.py

@@ -284,7 +284,7 @@ def writeCache(feed_uri, feed_info, data):
     if not os.path.exists(sources): os.makedirs(sources)
     xdoc=minidom.parseString('''<feed xmlns:planet="%s"
       xmlns="http://www.w3.org/2005/Atom"/>\n''' % planet.xmlns)
-    reconstitute.source(xdoc.documentElement,data.feed,data.bozo,data.version)
+    reconstitute.source(xdoc.documentElement,data.feed,data.bozo, data.get('version'))
     write(xdoc.toxml().encode('utf-8'), filename(sources, feed_uri))
     xdoc.unlink()
 

planet/splice.py

@@ -1,10 +1,13 @@
 """ Splice together a planet from a cache of feed entries """
-import glob, os, time, shutil
+import glob, os, time, shutil, pickle, traceback,sys
 from xml.dom import minidom
 import planet, config, feedparser, reconstitute, shell
 from reconstitute import createTextElement, date
 from spider import filename
 from planet import idindex
+import traceback
+
+posted_urls_file = 'posted_urls.pickle'
 
 def splice():
     """ Splice together a planet from a cache of entries """
@@ -62,20 +65,25 @@ def splice():
     sources = config.cache_sources_directory()
     for sub in config.subscriptions():
         data=feedparser.parse(filename(sources,sub))
-        if data.feed.has_key('id'): sub_ids.append(data.feed.id)
-        if not data.feed: continue
+        if data.feed.has_key('id'):
+	    sub_ids.append(data.feed.id)
+        if not data.feed:
+	    continue
 
         # warn on missing links
         if not data.feed.has_key('planet_message'):
-            if not data.feed.has_key('links'): data.feed['links'] = []
+            if not data.feed.has_key('links'):
+		data.feed['links'] = []
 
             for link in data.feed.links:
-              if link.rel == 'self': break
+              if link.rel == 'self':
+		  break
             else:
               log.debug('missing self link for ' + sub)
 
             for link in data.feed.links:
-              if link.rel == 'alternate' and 'html' in link.type: break
+              if link.rel == 'alternate' and 'html' in link.type:
+		  break
             else:
               log.debug('missing html link for ' + sub)
 
@@ -91,10 +99,22 @@ def splice():
     count = {}
     atomNS='http://www.w3.org/2005/Atom'
     new_feed_items = config.new_feed_items()
+
+    posted_urls = set()
+    if config.post_to_twitter():
+	if os.path.exists(posted_urls_file):
+	    try:
+		with open(posted_urls_file, 'rb') as f:
+		    posted_urls = pickle.load(f)
+            except Exception as ex:
+		log.error("Error reading posted_urls %s", ex)
+#    print(posted_urls)
+
     for mtime,file in dir:
         if index != None:
             base = os.path.basename(file)
-            if index.has_key(base) and index[base] not in sub_ids: continue
+            if index.has_key(base) and index[base] not in sub_ids:
+		continue
 
         try:
             entry=minidom.parse(file)
@@ -109,30 +129,81 @@ def splice():
                 if ids:
                     id = ids[0].childNodes[0].nodeValue
                     count[id] = count.get(id,0) + 1
-                    if new_feed_items and count[id] > new_feed_items: continue
+                    if new_feed_items and count[id] > new_feed_items:
+			continue
 
                     if id not in sub_ids:
                         ids = sources[0].getElementsByTagName('planet:id')
-                        if not ids: continue
+                        if not ids:
+			    continue
                         id = ids[0].childNodes[0].nodeValue
                         if id not in sub_ids:
                           log.warn('Skipping: ' + id)
-                        if id not in sub_ids: continue
+                        if id not in sub_ids:
+			    continue
+
+	    # Twitter integration
+	    if config.post_to_twitter():
+		url = None
+		twitter = None
+		title = "Untitled post..."
+		links = entry.getElementsByTagName('link')
+		if links:
+		    for link in links:
+			if link.hasAttribute('rel') and link.hasAttribute('type') and link.hasAttribute('href'):
+			    if (link.getAttribute('rel') == 'alternate' and
+				link.getAttribute('type') == 'text/html'):
+				url = link.getAttribute('href')
+				break
+
+		titles = entry.getElementsByTagName('title')
+		if titles:
+		    title = unicode(titles[0].firstChild.nodeValue.encode('utf-8'), 'utf-8').strip()
+		handles = entry.getElementsByTagName('planet:twitter')
+		if (handles):
+		    twitter = unicode(handles[0].firstChild.nodeValue.encode('utf-8'), "utf-8")
 
+		if url is not None and url not in posted_urls:
+#		    log.debug("Going to post URL to Twitter: twitter='{}' title='{}', url='{}'".format(twitter, title, url))
+		    txt_append = u''
+		    if twitter:
+			txt_append = u" (by " + ' & '.join(["@"+tw for tw in twitter.strip().split(',')]).encode('utf-8') + u")"
+		    max_title_len = 280 - 20 - len(txt_append)
+		    if (len(title) > max_title_len):
+			title = title[:max_title_len]
+		    txt =  title + txt_append + u"\n" + url
+
+		    log.debug(u"Text to post '{}'".format(txt))
+		    try:
+			posted_urls.add(url)
+			config.twitter_api.update_status(txt)
+		    except Exception as ex:
+			log.error(u"Error posting to Twitter: %s", ex)
+
             # add entry to feed
             feed.appendChild(entry.documentElement)
             items = items + 1
-            if items >= max_items: break
-        except:
-            log.error("Error parsing %s", file)
+            if items >= max_items:
+		break
+        except Exception as ex:
+            log.error("Error parsing %s: %s", file, ex)
+	    exc_type, exc_value, exc_traceback = sys.exc_info()
+	    traceback.print_exception(exc_type, exc_value, exc_traceback,
+				      limit=2, file=sys.stdout)
 
-    if index: index.close()
+    if config.post_to_twitter():
+	with open(posted_urls_file, 'wb') as f:
+	    pickle.dump(posted_urls, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+    if index:
+	index.close()
 
     return doc
 
 def apply(doc):
     output_dir = config.output_dir()
-    if not os.path.exists(output_dir): os.makedirs(output_dir)
+    if not os.path.exists(output_dir):
+	os.makedirs(output_dir)
     log = planet.logger
 
     planet_filters = config.filters('Planet')

requirements.txt

@@ -0,0 +1 @@
+tweepy